{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cd7e68e7-6d15-46e7-a5a9-985a8dfd5d1c",
   "metadata": {},
   "source": [
    "# Fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fffb59e5-00d5-4e68-b6af-98e301d2ab94",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please run\n",
      "\n",
      "python -m bitsandbytes\n",
      "\n",
      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "bin /data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32\n",
      "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.0\n",
      "CUDA SETUP: Detected CUDA version 117\n",
      "CUDA SETUP: Loading binary /data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable.\n",
      "  warn(\"The installed version of bitsandbytes was compiled without GPU support. \"\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/user/rajkumardheivanayahi/oauth_callback')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('servers!server=rajkumardheivanayahi/\", \"access'), PosixPath('[\"access'), PosixPath('servers!user=rajkumardheivanayahi\"]')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('http'), PosixPath('8081/hub/api'), PosixPath('//127.0.0.1')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('http'), PosixPath('//127.0.0.1'), PosixPath('8081/hub/api/users/rajkumardheivanayahi/activity')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/user/rajkumardheivanayahi')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('http'), PosixPath('49529/user/rajkumardheivanayahi'), PosixPath('//127.0.0.1')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//matplotlib_inline.backend_inline'), PosixPath('module')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/cuda/lib64')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: No libcudart.so found! Install CUDA or the cudatoolkit package (anaconda)!\n",
      "  warn(msg)\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoConfig, AutoTokenizer, TrainingArguments, AutoModel, AutoModelForSequenceClassification, Trainer, pipeline, DataCollatorForTokenClassification, AutoModelForTokenClassification, AutoModelForSeq2SeqLM, DataCollatorForSeq2Seq, Seq2SeqTrainingArguments, Seq2SeqTrainer\n",
    "from datasets import load_dataset, load_metric, Dataset\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils import resample\n",
    "from sklearn.metrics import roc_auc_score, f1_score, confusion_matrix, ConfusionMatrixDisplay, classification_report, accuracy_score\n",
    "import torch\n",
    "import torchinfo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b9e9c6cc-207d-4d25-9988-564c167891ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = 'bert-base-cased'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f26359cc-ef90-4793-bffc-35f9b0c92cd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d06475ea-4537-48df-a3fc-7e42f2bc96f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [101, 8667, 1291, 102], 'token_type_ids': [0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1]}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer(\"Hello World\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e7eee3ba-351f-4bf6-b350-7054dcda2be9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hello', 'world', 'full', 'of', 'Chi', '##mpo', '##nz', '##ees']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.tokenize(\"Hello world full of Chimponzees\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "67aca6e2-75eb-4497-99e3-4b7c09ed798d",
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = tokenizer.encode('Hello world')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "682aad1d-54c3-4ef3-9d36-f8cfc9ede5d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[CLS]', 'Hello', 'world', '[SEP]']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.convert_ids_to_tokens(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "39e19a86-d40c-4961-891d-3e5f7fd0a80b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./DATA.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d621f417-e0d2-4a0e-9b94-2c2ef16c8e2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Fwd Packets/s</th>\n",
       "      <th>Bwd Packets/s</th>\n",
       "      <th>Flow Bytes/s</th>\n",
       "      <th>packet</th>\n",
       "      <th>source_port</th>\n",
       "      <th>destination_port</th>\n",
       "      <th>IP len</th>\n",
       "      <th>payload len</th>\n",
       "      <th>IP ttl</th>\n",
       "      <th>IP tos</th>\n",
       "      <th>...</th>\n",
       "      <th>byte_492</th>\n",
       "      <th>byte_493</th>\n",
       "      <th>byte_494</th>\n",
       "      <th>byte_495</th>\n",
       "      <th>byte_496</th>\n",
       "      <th>byte_497</th>\n",
       "      <th>byte_498</th>\n",
       "      <th>byte_499</th>\n",
       "      <th>attack_cat</th>\n",
       "      <th>packet_dat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1170</td>\n",
       "      <td>1625</td>\n",
       "      <td>3406409</td>\n",
       "      <td>-1</td>\n",
       "      <td>80</td>\n",
       "      <td>50580</td>\n",
       "      <td>1500</td>\n",
       "      <td>1460</td>\n",
       "      <td>118</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>133</td>\n",
       "      <td>218</td>\n",
       "      <td>248</td>\n",
       "      <td>64</td>\n",
       "      <td>68</td>\n",
       "      <td>236</td>\n",
       "      <td>51</td>\n",
       "      <td>110</td>\n",
       "      <td>Normal</td>\n",
       "      <td>1170 1625 3406409 -1 80 50580 1500 1460 118 0 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1357</td>\n",
       "      <td>1905</td>\n",
       "      <td>3946827</td>\n",
       "      <td>-1</td>\n",
       "      <td>80</td>\n",
       "      <td>50580</td>\n",
       "      <td>1500</td>\n",
       "      <td>1460</td>\n",
       "      <td>118</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>133</td>\n",
       "      <td>218</td>\n",
       "      <td>248</td>\n",
       "      <td>64</td>\n",
       "      <td>68</td>\n",
       "      <td>236</td>\n",
       "      <td>51</td>\n",
       "      <td>110</td>\n",
       "      <td>Normal</td>\n",
       "      <td>1357 1905 3946827 -1 80 50580 1500 1460 118 0 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1223</td>\n",
       "      <td>1747</td>\n",
       "      <td>3661835</td>\n",
       "      <td>-1</td>\n",
       "      <td>80</td>\n",
       "      <td>50580</td>\n",
       "      <td>1500</td>\n",
       "      <td>1460</td>\n",
       "      <td>118</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>133</td>\n",
       "      <td>218</td>\n",
       "      <td>248</td>\n",
       "      <td>64</td>\n",
       "      <td>68</td>\n",
       "      <td>236</td>\n",
       "      <td>51</td>\n",
       "      <td>110</td>\n",
       "      <td>Normal</td>\n",
       "      <td>1223 1747 3661835 -1 80 50580 1500 1460 118 0 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1183</td>\n",
       "      <td>1610</td>\n",
       "      <td>3465091</td>\n",
       "      <td>-1</td>\n",
       "      <td>80</td>\n",
       "      <td>50580</td>\n",
       "      <td>1500</td>\n",
       "      <td>1460</td>\n",
       "      <td>118</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>133</td>\n",
       "      <td>218</td>\n",
       "      <td>248</td>\n",
       "      <td>64</td>\n",
       "      <td>68</td>\n",
       "      <td>236</td>\n",
       "      <td>51</td>\n",
       "      <td>110</td>\n",
       "      <td>Normal</td>\n",
       "      <td>1183 1610 3465091 -1 80 50580 1500 1460 118 0 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>464</td>\n",
       "      <td>655</td>\n",
       "      <td>1356121</td>\n",
       "      <td>-1</td>\n",
       "      <td>80</td>\n",
       "      <td>50580</td>\n",
       "      <td>1500</td>\n",
       "      <td>1460</td>\n",
       "      <td>118</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>133</td>\n",
       "      <td>218</td>\n",
       "      <td>248</td>\n",
       "      <td>64</td>\n",
       "      <td>68</td>\n",
       "      <td>236</td>\n",
       "      <td>51</td>\n",
       "      <td>110</td>\n",
       "      <td>Normal</td>\n",
       "      <td>464 655 1356121 -1 80 50580 1500 1460 118 0 5 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1700736</th>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>7028</td>\n",
       "      <td>-1</td>\n",
       "      <td>20503</td>\n",
       "      <td>33532</td>\n",
       "      <td>313</td>\n",
       "      <td>273</td>\n",
       "      <td>254</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>Backdoor</td>\n",
       "      <td>13 12 7028 -1 20503 33532 313 273 254 0 5 0 -1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1700737</th>\n",
       "      <td>24</td>\n",
       "      <td>19</td>\n",
       "      <td>2579</td>\n",
       "      <td>-1</td>\n",
       "      <td>1014</td>\n",
       "      <td>60251</td>\n",
       "      <td>148</td>\n",
       "      <td>108</td>\n",
       "      <td>253</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>Backdoor</td>\n",
       "      <td>24 19 2579 -1 1014 60251 148 108 253 0 5 0 -1 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1700738</th>\n",
       "      <td>24</td>\n",
       "      <td>19</td>\n",
       "      <td>2579</td>\n",
       "      <td>-1</td>\n",
       "      <td>1014</td>\n",
       "      <td>60251</td>\n",
       "      <td>148</td>\n",
       "      <td>108</td>\n",
       "      <td>252</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>Backdoor</td>\n",
       "      <td>24 19 2579 -1 1014 60251 148 108 252 0 5 0 -1 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1700739</th>\n",
       "      <td>24</td>\n",
       "      <td>19</td>\n",
       "      <td>2579</td>\n",
       "      <td>-1</td>\n",
       "      <td>60251</td>\n",
       "      <td>1014</td>\n",
       "      <td>54</td>\n",
       "      <td>14</td>\n",
       "      <td>255</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>Backdoor</td>\n",
       "      <td>24 19 2579 -1 60251 1014 54 14 255 0 5 0 -1 67...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1700740</th>\n",
       "      <td>24</td>\n",
       "      <td>19</td>\n",
       "      <td>2579</td>\n",
       "      <td>-1</td>\n",
       "      <td>60251</td>\n",
       "      <td>1014</td>\n",
       "      <td>54</td>\n",
       "      <td>14</td>\n",
       "      <td>254</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>Backdoor</td>\n",
       "      <td>24 19 2579 -1 60251 1014 54 14 254 0 5 0 -1 67...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1700741 rows × 515 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Fwd Packets/s  Bwd Packets/s  Flow Bytes/s  packet  source_port  \\\n",
       "0                 1170           1625       3406409      -1           80   \n",
       "1                 1357           1905       3946827      -1           80   \n",
       "2                 1223           1747       3661835      -1           80   \n",
       "3                 1183           1610       3465091      -1           80   \n",
       "4                  464            655       1356121      -1           80   \n",
       "...                ...            ...           ...     ...          ...   \n",
       "1700736             13             12          7028      -1        20503   \n",
       "1700737             24             19          2579      -1         1014   \n",
       "1700738             24             19          2579      -1         1014   \n",
       "1700739             24             19          2579      -1        60251   \n",
       "1700740             24             19          2579      -1        60251   \n",
       "\n",
       "         destination_port  IP len  payload len  IP ttl  IP tos  ...  byte_492  \\\n",
       "0                   50580    1500         1460     118       0  ...       133   \n",
       "1                   50580    1500         1460     118       0  ...       133   \n",
       "2                   50580    1500         1460     118       0  ...       133   \n",
       "3                   50580    1500         1460     118       0  ...       133   \n",
       "4                   50580    1500         1460     118       0  ...       133   \n",
       "...                   ...     ...          ...     ...     ...  ...       ...   \n",
       "1700736             33532     313          273     254       0  ...        -1   \n",
       "1700737             60251     148          108     253       0  ...        -1   \n",
       "1700738             60251     148          108     252       0  ...        -1   \n",
       "1700739              1014      54           14     255       0  ...        -1   \n",
       "1700740              1014      54           14     254       0  ...        -1   \n",
       "\n",
       "         byte_493  byte_494  byte_495  byte_496  byte_497  byte_498  byte_499  \\\n",
       "0             218       248        64        68       236        51       110   \n",
       "1             218       248        64        68       236        51       110   \n",
       "2             218       248        64        68       236        51       110   \n",
       "3             218       248        64        68       236        51       110   \n",
       "4             218       248        64        68       236        51       110   \n",
       "...           ...       ...       ...       ...       ...       ...       ...   \n",
       "1700736        -1        -1        -1        -1        -1        -1        -1   \n",
       "1700737        -1        -1        -1        -1        -1        -1        -1   \n",
       "1700738        -1        -1        -1        -1        -1        -1        -1   \n",
       "1700739        -1        -1        -1        -1        -1        -1        -1   \n",
       "1700740        -1        -1        -1        -1        -1        -1        -1   \n",
       "\n",
       "         attack_cat                                         packet_dat  \n",
       "0            Normal  1170 1625 3406409 -1 80 50580 1500 1460 118 0 ...  \n",
       "1            Normal  1357 1905 3946827 -1 80 50580 1500 1460 118 0 ...  \n",
       "2            Normal  1223 1747 3661835 -1 80 50580 1500 1460 118 0 ...  \n",
       "3            Normal  1183 1610 3465091 -1 80 50580 1500 1460 118 0 ...  \n",
       "4            Normal  464 655 1356121 -1 80 50580 1500 1460 118 0 5 ...  \n",
       "...             ...                                                ...  \n",
       "1700736    Backdoor  13 12 7028 -1 20503 33532 313 273 254 0 5 0 -1...  \n",
       "1700737    Backdoor  24 19 2579 -1 1014 60251 148 108 253 0 5 0 -1 ...  \n",
       "1700738    Backdoor  24 19 2579 -1 1014 60251 148 108 252 0 5 0 -1 ...  \n",
       "1700739    Backdoor  24 19 2579 -1 60251 1014 54 14 255 0 5 0 -1 67...  \n",
       "1700740    Backdoor  24 19 2579 -1 60251 1014 54 14 254 0 5 0 -1 67...  \n",
       "\n",
       "[1700741 rows x 515 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "871d05c2-36ad-43ef-857c-8d9e26dff131",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[['packet_dat', 'attack_cat']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2eca0b0f-857b-4642-80ef-59b8e0ef35ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop_duplicates(subset=['packet_dat'], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "011b289b-f019-4279-a460-b8a4b9ab730f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packet_dat</th>\n",
       "      <th>attack_cat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1170 1625 3406409 -1 80 50580 1500 1460 118 0 ...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1357 1905 3946827 -1 80 50580 1500 1460 118 0 ...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1223 1747 3661835 -1 80 50580 1500 1460 118 0 ...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1183 1610 3465091 -1 80 50580 1500 1460 118 0 ...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>464 655 1356121 -1 80 50580 1500 1460 118 0 5 ...</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1696826</th>\n",
       "      <td>13 12 7028 -1 20503 33532 313 273 254 0 5 0 -1...</td>\n",
       "      <td>Backdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1696827</th>\n",
       "      <td>24 19 2579 -1 1014 60251 148 108 253 0 5 0 -1 ...</td>\n",
       "      <td>Backdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1696828</th>\n",
       "      <td>24 19 2579 -1 1014 60251 148 108 252 0 5 0 -1 ...</td>\n",
       "      <td>Backdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1696829</th>\n",
       "      <td>24 19 2579 -1 60251 1014 54 14 255 0 5 0 -1 67...</td>\n",
       "      <td>Backdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1696830</th>\n",
       "      <td>24 19 2579 -1 60251 1014 54 14 254 0 5 0 -1 67...</td>\n",
       "      <td>Backdoor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1696831 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                packet_dat attack_cat\n",
       "0        1170 1625 3406409 -1 80 50580 1500 1460 118 0 ...     Normal\n",
       "1        1357 1905 3946827 -1 80 50580 1500 1460 118 0 ...     Normal\n",
       "2        1223 1747 3661835 -1 80 50580 1500 1460 118 0 ...     Normal\n",
       "3        1183 1610 3465091 -1 80 50580 1500 1460 118 0 ...     Normal\n",
       "4        464 655 1356121 -1 80 50580 1500 1460 118 0 5 ...     Normal\n",
       "...                                                    ...        ...\n",
       "1696826  13 12 7028 -1 20503 33532 313 273 254 0 5 0 -1...   Backdoor\n",
       "1696827  24 19 2579 -1 1014 60251 148 108 253 0 5 0 -1 ...   Backdoor\n",
       "1696828  24 19 2579 -1 1014 60251 148 108 252 0 5 0 -1 ...   Backdoor\n",
       "1696829  24 19 2579 -1 60251 1014 54 14 255 0 5 0 -1 67...   Backdoor\n",
       "1696830  24 19 2579 -1 60251 1014 54 14 254 0 5 0 -1 67...   Backdoor\n",
       "\n",
       "[1696831 rows x 2 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "927ac4d5-d0dc-4e86-95f8-948e70ac5e14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DoS Hulk                      596942\n",
       "Normal                        259650\n",
       "DDoS                          182866\n",
       "Heartbleed                    167360\n",
       "SSH Patator                    85397\n",
       "DoS                            70239\n",
       "Exploits                       65551\n",
       "Generic                        49120\n",
       "FTP Patator                    45189\n",
       "Fuzzers                        36198\n",
       "Reconnaissance                 34645\n",
       "DoS GoldenEye                  33486\n",
       "Web Attack - Brute Force       18630\n",
       "Infiltration                   10774\n",
       "Worms                           9466\n",
       "DoS SlowHTTPTest                7999\n",
       "DoS Slowloris                   6276\n",
       "Web Attack - XSS                5154\n",
       "Bot                             5139\n",
       "Analysis                        2612\n",
       "Backdoor                        1752\n",
       "Shellcode                       1498\n",
       "Port Scan                        843\n",
       "Web Attack - SQL Injection        45\n",
       "Name: attack_cat, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.attack_cat.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2fb7d42e-8d44-412d-b0e4-f9f3c3eb6bad",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df, test_df = train_test_split(df, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "fe8bcf2a-8b1d-4fb1-a2e9-40da5e1d51b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DoS Hulk                      417967\n",
       "Normal                        181742\n",
       "DDoS                          127975\n",
       "Heartbleed                    117056\n",
       "SSH Patator                    59905\n",
       "DoS                            49047\n",
       "Exploits                       45931\n",
       "Generic                        34343\n",
       "FTP Patator                    31684\n",
       "Fuzzers                        25409\n",
       "Reconnaissance                 24250\n",
       "DoS GoldenEye                  23355\n",
       "Web Attack - Brute Force       13014\n",
       "Infiltration                    7514\n",
       "Worms                           6552\n",
       "DoS SlowHTTPTest                5567\n",
       "DoS Slowloris                   4424\n",
       "Bot                             3679\n",
       "Web Attack - XSS                3605\n",
       "Analysis                        1819\n",
       "Backdoor                        1245\n",
       "Shellcode                       1063\n",
       "Port Scan                        605\n",
       "Web Attack - SQL Injection        30\n",
       "Name: attack_cat, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.attack_cat.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6a305375-d128-4f5e-9d6d-e46cbcb0b9cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DoS Hulk                      178975\n",
       "Normal                         77908\n",
       "DDoS                           54891\n",
       "Heartbleed                     50304\n",
       "SSH Patator                    25492\n",
       "DoS                            21192\n",
       "Exploits                       19620\n",
       "Generic                        14777\n",
       "FTP Patator                    13505\n",
       "Fuzzers                        10789\n",
       "Reconnaissance                 10395\n",
       "DoS GoldenEye                  10131\n",
       "Web Attack - Brute Force        5616\n",
       "Infiltration                    3260\n",
       "Worms                           2914\n",
       "DoS SlowHTTPTest                2432\n",
       "DoS Slowloris                   1852\n",
       "Web Attack - XSS                1549\n",
       "Bot                             1460\n",
       "Analysis                         793\n",
       "Backdoor                         507\n",
       "Shellcode                        435\n",
       "Port Scan                        238\n",
       "Web Attack - SQL Injection        15\n",
       "Name: attack_cat, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.attack_cat.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e6122cd2-c111-4790-a00b-4a77524fe38c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.to_csv('./DATA_TRAIN.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c0bfadbf-05ca-490a-9aa6-89c45eb627bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df.to_csv('./DATA_TEST.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4c3299d2-3bce-4ea3-b047-1b5aee006073",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = train_df.groupby('attack_cat').apply(lambda x: x.sample(min(len(x), 100000))).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "06552d55-9dc7-4c52-a81c-a584f829b186",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DDoS                          100000\n",
       "DoS Hulk                      100000\n",
       "Normal                        100000\n",
       "Heartbleed                    100000\n",
       "SSH Patator                    59905\n",
       "DoS                            49047\n",
       "Exploits                       45931\n",
       "Generic                        34343\n",
       "FTP Patator                    31684\n",
       "Fuzzers                        25409\n",
       "Reconnaissance                 24250\n",
       "DoS GoldenEye                  23355\n",
       "Web Attack - Brute Force       13014\n",
       "Infiltration                    7514\n",
       "Worms                           6552\n",
       "DoS SlowHTTPTest                5567\n",
       "DoS Slowloris                   4424\n",
       "Bot                             3679\n",
       "Web Attack - XSS                3605\n",
       "Analysis                        1819\n",
       "Backdoor                        1245\n",
       "Shellcode                       1063\n",
       "Port Scan                        605\n",
       "Web Attack - SQL Injection        30\n",
       "Name: attack_cat, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.attack_cat.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f2f2418c-3035-4cb0-a5c7-207af63bf8c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "ros = RandomOverSampler(random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "058af95e-5f23-4240-9150-4da6adb1ded4",
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = ros.fit_resample(train_df[['packet_dat']], train_df[['attack_cat']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5b32bbcf-db09-4ca4-afe6-cafb1771f50b",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([X,y], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "7c606ec9-d627-4fa0-b3a9-396d11364784",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packet_dat</th>\n",
       "      <th>attack_cat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14 11 2798 -1 62088 80 229 189 62 0 5 3 -1 71 ...</td>\n",
       "      <td>Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24 19 4987 -1 80 63691 473 433 253 0 5 0 -1 72...</td>\n",
       "      <td>Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8 8 1637 -1 63648 80 230 190 63 0 5 3 -1 71 69...</td>\n",
       "      <td>Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30 24 6404 -1 80 65401 512 472 253 0 5 0 -1 72...</td>\n",
       "      <td>Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>21 17 4740 -1 13591 80 229 189 63 0 5 3 -1 71 ...</td>\n",
       "      <td>Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399995</th>\n",
       "      <td>12 70 91946 -1 80 33820 1500 1460 253 0 5 0 -1...</td>\n",
       "      <td>Worms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399996</th>\n",
       "      <td>14 77 103183 -1 80 49608 1500 1460 252 0 5 0 -...</td>\n",
       "      <td>Worms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399997</th>\n",
       "      <td>14 87 119898 -1 80 63800 1500 1460 252 0 5 0 -...</td>\n",
       "      <td>Worms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399998</th>\n",
       "      <td>18 108 141828 -1 80 54215 1500 1460 253 0 5 0 ...</td>\n",
       "      <td>Worms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399999</th>\n",
       "      <td>14 80 104802 -1 80 4298 41 1 252 0 5 0 -1 66 -...</td>\n",
       "      <td>Worms</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2400000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                packet_dat attack_cat\n",
       "0        14 11 2798 -1 62088 80 229 189 62 0 5 3 -1 71 ...   Analysis\n",
       "1        24 19 4987 -1 80 63691 473 433 253 0 5 0 -1 72...   Analysis\n",
       "2        8 8 1637 -1 63648 80 230 190 63 0 5 3 -1 71 69...   Analysis\n",
       "3        30 24 6404 -1 80 65401 512 472 253 0 5 0 -1 72...   Analysis\n",
       "4        21 17 4740 -1 13591 80 229 189 63 0 5 3 -1 71 ...   Analysis\n",
       "...                                                    ...        ...\n",
       "2399995  12 70 91946 -1 80 33820 1500 1460 253 0 5 0 -1...      Worms\n",
       "2399996  14 77 103183 -1 80 49608 1500 1460 252 0 5 0 -...      Worms\n",
       "2399997  14 87 119898 -1 80 63800 1500 1460 252 0 5 0 -...      Worms\n",
       "2399998  18 108 141828 -1 80 54215 1500 1460 253 0 5 0 ...      Worms\n",
       "2399999  14 80 104802 -1 80 4298 41 1 252 0 5 0 -1 66 -...      Worms\n",
       "\n",
       "[2400000 rows x 2 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d302f288-bfda-4ee6-82a5-97a795c37c24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Analysis                      100000\n",
       "Backdoor                      100000\n",
       "Web Attack - XSS              100000\n",
       "Web Attack - SQL Injection    100000\n",
       "Web Attack - Brute Force      100000\n",
       "Shellcode                     100000\n",
       "SSH Patator                   100000\n",
       "Reconnaissance                100000\n",
       "Port Scan                     100000\n",
       "Normal                        100000\n",
       "Infiltration                  100000\n",
       "Heartbleed                    100000\n",
       "Generic                       100000\n",
       "Fuzzers                       100000\n",
       "FTP Patator                   100000\n",
       "Exploits                      100000\n",
       "DoS Slowloris                 100000\n",
       "DoS SlowHTTPTest              100000\n",
       "DoS Hulk                      100000\n",
       "DoS GoldenEye                 100000\n",
       "DoS                           100000\n",
       "DDoS                          100000\n",
       "Bot                           100000\n",
       "Worms                         100000\n",
       "Name: attack_cat, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.attack_cat.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f04fbdca-8613-432f-bc7d-82cada4dafd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = data['attack_cat'].unique()\n",
    "\n",
    "target_map = {class_name: index for index, class_name in enumerate(sorted(classes))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "5486973c-4d66-4f9e-b766-86ac182d7ea0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Analysis': 0,\n",
       " 'Backdoor': 1,\n",
       " 'Bot': 2,\n",
       " 'DDoS': 3,\n",
       " 'DoS': 4,\n",
       " 'DoS GoldenEye': 5,\n",
       " 'DoS Hulk': 6,\n",
       " 'DoS SlowHTTPTest': 7,\n",
       " 'DoS Slowloris': 8,\n",
       " 'Exploits': 9,\n",
       " 'FTP Patator': 10,\n",
       " 'Fuzzers': 11,\n",
       " 'Generic': 12,\n",
       " 'Heartbleed': 13,\n",
       " 'Infiltration': 14,\n",
       " 'Normal': 15,\n",
       " 'Port Scan': 16,\n",
       " 'Reconnaissance': 17,\n",
       " 'SSH Patator': 18,\n",
       " 'Shellcode': 19,\n",
       " 'Web Attack - Brute Force': 20,\n",
       " 'Web Attack - SQL Injection': 21,\n",
       " 'Web Attack - XSS': 22,\n",
       " 'Worms': 23}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "8d20c3a0-fe81-49fa-87e2-f3dc8eadcb62",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['target'] = data['attack_cat'].map(target_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e205961e-d912-41f7-8485-3d48164f6bb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packet_dat</th>\n",
       "      <th>attack_cat</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14 11 2798 -1 62088 80 229 189 62 0 5 3 -1 71 ...</td>\n",
       "      <td>Analysis</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24 19 4987 -1 80 63691 473 433 253 0 5 0 -1 72...</td>\n",
       "      <td>Analysis</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8 8 1637 -1 63648 80 230 190 63 0 5 3 -1 71 69...</td>\n",
       "      <td>Analysis</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30 24 6404 -1 80 65401 512 472 253 0 5 0 -1 72...</td>\n",
       "      <td>Analysis</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>21 17 4740 -1 13591 80 229 189 63 0 5 3 -1 71 ...</td>\n",
       "      <td>Analysis</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399995</th>\n",
       "      <td>12 70 91946 -1 80 33820 1500 1460 253 0 5 0 -1...</td>\n",
       "      <td>Worms</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399996</th>\n",
       "      <td>14 77 103183 -1 80 49608 1500 1460 252 0 5 0 -...</td>\n",
       "      <td>Worms</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399997</th>\n",
       "      <td>14 87 119898 -1 80 63800 1500 1460 252 0 5 0 -...</td>\n",
       "      <td>Worms</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399998</th>\n",
       "      <td>18 108 141828 -1 80 54215 1500 1460 253 0 5 0 ...</td>\n",
       "      <td>Worms</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399999</th>\n",
       "      <td>14 80 104802 -1 80 4298 41 1 252 0 5 0 -1 66 -...</td>\n",
       "      <td>Worms</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2400000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                packet_dat attack_cat  target\n",
       "0        14 11 2798 -1 62088 80 229 189 62 0 5 3 -1 71 ...   Analysis       0\n",
       "1        24 19 4987 -1 80 63691 473 433 253 0 5 0 -1 72...   Analysis       0\n",
       "2        8 8 1637 -1 63648 80 230 190 63 0 5 3 -1 71 69...   Analysis       0\n",
       "3        30 24 6404 -1 80 65401 512 472 253 0 5 0 -1 72...   Analysis       0\n",
       "4        21 17 4740 -1 13591 80 229 189 63 0 5 3 -1 71 ...   Analysis       0\n",
       "...                                                    ...        ...     ...\n",
       "2399995  12 70 91946 -1 80 33820 1500 1460 253 0 5 0 -1...      Worms      23\n",
       "2399996  14 77 103183 -1 80 49608 1500 1460 252 0 5 0 -...      Worms      23\n",
       "2399997  14 87 119898 -1 80 63800 1500 1460 252 0 5 0 -...      Worms      23\n",
       "2399998  18 108 141828 -1 80 54215 1500 1460 253 0 5 0 ...      Worms      23\n",
       "2399999  14 80 104802 -1 80 4298 41 1 252 0 5 0 -1 66 -...      Worms      23\n",
       "\n",
       "[2400000 rows x 3 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a359276e-3fa7-43bd-9c90-374d1bbbdf5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[['packet_dat', 'target']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "54635877-9665-4b36-9ed7-e34bcba0a7cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.columns = ['packet', 'label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "39f15ae5-547f-4e00-869e-33c836e2498d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packet</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14 11 2798 -1 62088 80 229 189 62 0 5 3 -1 71 ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>24 19 4987 -1 80 63691 473 433 253 0 5 0 -1 72...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8 8 1637 -1 63648 80 230 190 63 0 5 3 -1 71 69...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30 24 6404 -1 80 65401 512 472 253 0 5 0 -1 72...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>21 17 4740 -1 13591 80 229 189 63 0 5 3 -1 71 ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399995</th>\n",
       "      <td>12 70 91946 -1 80 33820 1500 1460 253 0 5 0 -1...</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399996</th>\n",
       "      <td>14 77 103183 -1 80 49608 1500 1460 252 0 5 0 -...</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399997</th>\n",
       "      <td>14 87 119898 -1 80 63800 1500 1460 252 0 5 0 -...</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399998</th>\n",
       "      <td>18 108 141828 -1 80 54215 1500 1460 253 0 5 0 ...</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399999</th>\n",
       "      <td>14 80 104802 -1 80 4298 41 1 252 0 5 0 -1 66 -...</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2400000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    packet  label\n",
       "0        14 11 2798 -1 62088 80 229 189 62 0 5 3 -1 71 ...      0\n",
       "1        24 19 4987 -1 80 63691 473 433 253 0 5 0 -1 72...      0\n",
       "2        8 8 1637 -1 63648 80 230 190 63 0 5 3 -1 71 69...      0\n",
       "3        30 24 6404 -1 80 65401 512 472 253 0 5 0 -1 72...      0\n",
       "4        21 17 4740 -1 13591 80 229 189 63 0 5 3 -1 71 ...      0\n",
       "...                                                    ...    ...\n",
       "2399995  12 70 91946 -1 80 33820 1500 1460 253 0 5 0 -1...     23\n",
       "2399996  14 77 103183 -1 80 49608 1500 1460 252 0 5 0 -...     23\n",
       "2399997  14 87 119898 -1 80 63800 1500 1460 252 0 5 0 -...     23\n",
       "2399998  18 108 141828 -1 80 54215 1500 1460 253 0 5 0 ...     23\n",
       "2399999  14 80 104802 -1 80 4298 41 1 252 0 5 0 -1 66 -...     23\n",
       "\n",
       "[2400000 rows x 2 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "291869de-aa8e-496d-b249-128a33e10cb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_dataset = Dataset.from_pandas(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "7d288255-3814-4cff-a45c-9c15d989fad3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['packet', 'label'],\n",
       "    num_rows: 2400000\n",
       "})"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "b96d2b7b-9f5d-4f4c-8699-f92c9c7e6be1",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_dataset = raw_dataset.train_test_split(test_size=0.3, seed=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "4f5d598b-4bbe-4759-b1b9-a2065b522abe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['packet', 'label'],\n",
       "        num_rows: 1680000\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['packet', 'label'],\n",
       "        num_rows: 720000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "9ece6ef2-0d8a-46d4-a221-0d6dddc55de2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'packet': '12 74 103446 -1 80 32283 1500 1460 253 0 5 0 -1 104 154 241 129 0 196 192 141 87 156 55 79 75 21 37 12 221 156 195 186 244 109 63 40 142 167 137 151 4 92 61 62 55 210 40 32 8 150 219 248 14 72 227 14 54 141 178 62 207 137 118 70 225 202 192 232 232 85 51 121 107 233 228 144 69 246 166 219 114 105 25 233 190 9 244 236 88 203 97 81 99 237 31 78 188 128 1 188 173 162 190 213 53 54 178 255 31 251 119 148 7 109 195 242 128 58 142 235 183 162 228 122 187 244 233 143 9 16 237 235 149 96 197 12 81 191 166 243 76 243 52 0 240 180 116 47 62 186 84 177 177 149 41 121 228 16 107 87 76 54 41 162 180 222 19 148 92 167 94 247 254 242 210 189 171 54 171 199 121 143 121 122 175 235 115 10 160 237 227 141 118 10 203 192 102 128 195 188 151 246 180 136 114 19 241 8 59 100 89 39 23 109 226 149 95 26 235 86 192 0 3 142 35 32 171 193 227 249 179 22 210 185 132 104 56 4 18 58 128 12 222 144 28 235 129 3 169 44 30 74 152 113 157 124 253 206 197 210 18 63 58 120 181 181 154 245 102 161 226 255 136 248 107 2 140 236 138 71 53 217 57 233 146 246 95 167 235 127 147 233 94 255 245 175 141 247 192 31 62 233 118 122 87 52 214 223 234 154 188 169 14 1 248 142 0 0 0 167 33 234 40 1 58 193 192 177 134 100 62 247 183 235 173 126 191 95 175 215 235 245 250 253 126 191 95 175 215 235 245 250 92 212 96 77 116 62 94 125 75 132 239 92 68 198 210 147 50 97 51 177 28 152 76 201 132 206 196 114 97 51 38 19 59 17 201 132 204 152 76 236 71 38 19 50 97 51 177 28 152 76 201 132 206 196 114 97 51 38 19 59 17 201 132 204 152 76 236 71 38 19 50 97 51 177 28 152 76 201 132 206 196 114 97 51 38 19 59 17 201 132 204 152 76 236 71 38 19 50 97 51 177 28 152 76 201 132 206 196 114 97 51 38 19 59 17 201 132 204 152 76 236 71 38 19 50 97 51 177 28 152 76 185 51 2 228 0 0 0 239 33 1 47 168 160 4 227 60 79 155 203 150 17 240 188 4 15 128 64 124 3',\n",
       " 'label': 12}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_dataset['train'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "4b9d81be-d1b8-4f07-9dd0-c19a432aaa1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_batch(batch):\n",
    "  return tokenizer(batch['packet'], truncation=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "57701900-3ea1-4860-8524-4f5c80757f64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1680000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/720000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenized_datasets = raw_dataset.map(function=tokenize_batch, batched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "80be7cc1-f1e0-4aee-b27f-d75679e83e2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['packet', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "        num_rows: 1680000\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['packet', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
       "        num_rows: 720000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "1c7b7e1f-256c-4736-a755-cf94965277cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "config = AutoConfig.from_pretrained(checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "5c687020-f3ed-410f-a997-f480ec537ee9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertConfig {\n",
       "  \"_name_or_path\": \"bert-base-cased\",\n",
       "  \"architectures\": [\n",
       "    \"BertForMaskedLM\"\n",
       "  ],\n",
       "  \"attention_probs_dropout_prob\": 0.1,\n",
       "  \"classifier_dropout\": null,\n",
       "  \"gradient_checkpointing\": false,\n",
       "  \"hidden_act\": \"gelu\",\n",
       "  \"hidden_dropout_prob\": 0.1,\n",
       "  \"hidden_size\": 768,\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"intermediate_size\": 3072,\n",
       "  \"layer_norm_eps\": 1e-12,\n",
       "  \"max_position_embeddings\": 512,\n",
       "  \"model_type\": \"bert\",\n",
       "  \"num_attention_heads\": 12,\n",
       "  \"num_hidden_layers\": 12,\n",
       "  \"pad_token_id\": 0,\n",
       "  \"position_embedding_type\": \"absolute\",\n",
       "  \"transformers_version\": \"4.29.2\",\n",
       "  \"type_vocab_size\": 2,\n",
       "  \"use_cache\": true,\n",
       "  \"vocab_size\": 28996\n",
       "}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8faec3a3-e609-4d89-85c6-32862cfff27b",
   "metadata": {},
   "outputs": [],
   "source": [
    "config.label2id = target_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "07da8709-e90b-4803-906c-7abb0bbc7b2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "config.id2label = {v:k for k,v in target_map.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "9e0e462f-2cb0-4f3a-8c0d-7f76b837d5d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertConfig {\n",
       "  \"_name_or_path\": \"bert-base-cased\",\n",
       "  \"architectures\": [\n",
       "    \"BertForMaskedLM\"\n",
       "  ],\n",
       "  \"attention_probs_dropout_prob\": 0.1,\n",
       "  \"classifier_dropout\": null,\n",
       "  \"gradient_checkpointing\": false,\n",
       "  \"hidden_act\": \"gelu\",\n",
       "  \"hidden_dropout_prob\": 0.1,\n",
       "  \"hidden_size\": 768,\n",
       "  \"id2label\": {\n",
       "    \"0\": \"Analysis\",\n",
       "    \"1\": \"Backdoor\",\n",
       "    \"2\": \"Bot\",\n",
       "    \"3\": \"DDoS\",\n",
       "    \"4\": \"DoS\",\n",
       "    \"5\": \"DoS GoldenEye\",\n",
       "    \"6\": \"DoS Hulk\",\n",
       "    \"7\": \"DoS SlowHTTPTest\",\n",
       "    \"8\": \"DoS Slowloris\",\n",
       "    \"9\": \"Exploits\",\n",
       "    \"10\": \"FTP Patator\",\n",
       "    \"11\": \"Fuzzers\",\n",
       "    \"12\": \"Generic\",\n",
       "    \"13\": \"Heartbleed\",\n",
       "    \"14\": \"Infiltration\",\n",
       "    \"15\": \"Normal\",\n",
       "    \"16\": \"Port Scan\",\n",
       "    \"17\": \"Reconnaissance\",\n",
       "    \"18\": \"SSH Patator\",\n",
       "    \"19\": \"Shellcode\",\n",
       "    \"20\": \"Web Attack - Brute Force\",\n",
       "    \"21\": \"Web Attack - SQL Injection\",\n",
       "    \"22\": \"Web Attack - XSS\",\n",
       "    \"23\": \"Worms\"\n",
       "  },\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"intermediate_size\": 3072,\n",
       "  \"label2id\": {\n",
       "    \"Analysis\": 0,\n",
       "    \"Backdoor\": 1,\n",
       "    \"Bot\": 2,\n",
       "    \"DDoS\": 3,\n",
       "    \"DoS\": 4,\n",
       "    \"DoS GoldenEye\": 5,\n",
       "    \"DoS Hulk\": 6,\n",
       "    \"DoS SlowHTTPTest\": 7,\n",
       "    \"DoS Slowloris\": 8,\n",
       "    \"Exploits\": 9,\n",
       "    \"FTP Patator\": 10,\n",
       "    \"Fuzzers\": 11,\n",
       "    \"Generic\": 12,\n",
       "    \"Heartbleed\": 13,\n",
       "    \"Infiltration\": 14,\n",
       "    \"Normal\": 15,\n",
       "    \"Port Scan\": 16,\n",
       "    \"Reconnaissance\": 17,\n",
       "    \"SSH Patator\": 18,\n",
       "    \"Shellcode\": 19,\n",
       "    \"Web Attack - Brute Force\": 20,\n",
       "    \"Web Attack - SQL Injection\": 21,\n",
       "    \"Web Attack - XSS\": 22,\n",
       "    \"Worms\": 23\n",
       "  },\n",
       "  \"layer_norm_eps\": 1e-12,\n",
       "  \"max_position_embeddings\": 512,\n",
       "  \"model_type\": \"bert\",\n",
       "  \"num_attention_heads\": 12,\n",
       "  \"num_hidden_layers\": 12,\n",
       "  \"pad_token_id\": 0,\n",
       "  \"position_embedding_type\": \"absolute\",\n",
       "  \"transformers_version\": \"4.29.2\",\n",
       "  \"type_vocab_size\": 2,\n",
       "  \"use_cache\": true,\n",
       "  \"vocab_size\": 28996\n",
       "}"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "2c7f9359-4931-4013-8740-30943df96efa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = AutoModelForSequenceClassification.from_pretrained(checkpoint, config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "20621440-bac6-46a3-ae13-f02d3ea3b218",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "================================================================================\n",
       "Layer (type:depth-idx)                                  Param #\n",
       "================================================================================\n",
       "BertForSequenceClassification                           --\n",
       "├─BertModel: 1-1                                        --\n",
       "│    └─BertEmbeddings: 2-1                              --\n",
       "│    │    └─Embedding: 3-1                              22,268,928\n",
       "│    │    └─Embedding: 3-2                              393,216\n",
       "│    │    └─Embedding: 3-3                              1,536\n",
       "│    │    └─LayerNorm: 3-4                              1,536\n",
       "│    │    └─Dropout: 3-5                                --\n",
       "│    └─BertEncoder: 2-2                                 --\n",
       "│    │    └─ModuleList: 3-6                             85,054,464\n",
       "│    └─BertPooler: 2-3                                  --\n",
       "│    │    └─Linear: 3-7                                 590,592\n",
       "│    │    └─Tanh: 3-8                                   --\n",
       "├─Dropout: 1-2                                          --\n",
       "├─Linear: 1-3                                           18,456\n",
       "================================================================================\n",
       "Total params: 108,328,728\n",
       "Trainable params: 108,328,728\n",
       "Non-trainable params: 0\n",
       "================================================================================"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torchinfo.summary(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "1a1dadc4-9442-418f-b3f6-05918be179bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/torch/cuda/__init__.py:546: UserWarning: Can't initialize NVML\n",
      "  warnings.warn(\"Can't initialize NVML\")\n"
     ]
    }
   ],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir='BERT',\n",
    "    evaluation_strategy='epoch',\n",
    "    save_strategy='epoch',\n",
    "    num_train_epochs=5,\n",
    "    per_device_train_batch_size=64,\n",
    "    per_device_eval_batch_size=128\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "7f826f1d-ca8f-4700-ae76-b6d9d34c63d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metric(logits_and_labels):\n",
    "  logits, labels = logits_and_labels\n",
    "  predictions = np.argmax(logits, axis=-1)\n",
    "  acc = np.mean(predictions == labels)\n",
    "  f1 = f1_score(labels, predictions, average='macro')\n",
    "  return {'accuracy':acc, 'f1-score':f1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "b2ddbdc2-d366-4fa4-819c-e097f527330b",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_datasets['train'],\n",
    "    eval_dataset=tokenized_datasets['test'],\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics = compute_metric\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f7b8be27-0739-4227-866b-7443c216d7e3",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'trainer' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m~/ipykernel_1711032/4032920361.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'trainer' is not defined"
     ]
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "63c6b9d1-2319-48e5-994d-625f73558004",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<transformers.trainer.Trainer at 0x7fcd725442b0>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "56bc648d-1faf-42b9-910d-6ace67df70ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'loss': 0.7694,\n",
       "  'learning_rate': 4.966644429619747e-05,\n",
       "  'epoch': 0.03,\n",
       "  'step': 500},\n",
       " {'loss': 0.1494,\n",
       "  'learning_rate': 4.933288859239493e-05,\n",
       "  'epoch': 0.07,\n",
       "  'step': 1000},\n",
       " {'loss': 0.1019,\n",
       "  'learning_rate': 4.89993328885924e-05,\n",
       "  'epoch': 0.1,\n",
       "  'step': 1500},\n",
       " {'loss': 0.0828,\n",
       "  'learning_rate': 4.866577718478986e-05,\n",
       "  'epoch': 0.13,\n",
       "  'step': 2000},\n",
       " {'loss': 0.0724,\n",
       "  'learning_rate': 4.833222148098733e-05,\n",
       "  'epoch': 0.17,\n",
       "  'step': 2500},\n",
       " {'loss': 0.0656,\n",
       "  'learning_rate': 4.799866577718479e-05,\n",
       "  'epoch': 0.2,\n",
       "  'step': 3000},\n",
       " {'loss': 0.0598,\n",
       "  'learning_rate': 4.766511007338226e-05,\n",
       "  'epoch': 0.23,\n",
       "  'step': 3500},\n",
       " {'loss': 0.0585,\n",
       "  'learning_rate': 4.733155436957972e-05,\n",
       "  'epoch': 0.27,\n",
       "  'step': 4000},\n",
       " {'loss': 0.0563,\n",
       "  'learning_rate': 4.699799866577719e-05,\n",
       "  'epoch': 0.3,\n",
       "  'step': 4500},\n",
       " {'loss': 0.0513,\n",
       "  'learning_rate': 4.6664442961974656e-05,\n",
       "  'epoch': 0.33,\n",
       "  'step': 5000},\n",
       " {'loss': 0.0488,\n",
       "  'learning_rate': 4.633088725817212e-05,\n",
       "  'epoch': 0.37,\n",
       "  'step': 5500},\n",
       " {'loss': 0.0466,\n",
       "  'learning_rate': 4.5997331554369585e-05,\n",
       "  'epoch': 0.4,\n",
       "  'step': 6000},\n",
       " {'loss': 0.047,\n",
       "  'learning_rate': 4.5663775850567046e-05,\n",
       "  'epoch': 0.43,\n",
       "  'step': 6500},\n",
       " {'loss': 0.0443,\n",
       "  'learning_rate': 4.5330220146764514e-05,\n",
       "  'epoch': 0.47,\n",
       "  'step': 7000},\n",
       " {'loss': 0.0429,\n",
       "  'learning_rate': 4.4996664442961976e-05,\n",
       "  'epoch': 0.5,\n",
       "  'step': 7500},\n",
       " {'loss': 0.0419,\n",
       "  'learning_rate': 4.4663108739159444e-05,\n",
       "  'epoch': 0.53,\n",
       "  'step': 8000},\n",
       " {'loss': 0.0417,\n",
       "  'learning_rate': 4.4329553035356905e-05,\n",
       "  'epoch': 0.57,\n",
       "  'step': 8500},\n",
       " {'loss': 0.0402,\n",
       "  'learning_rate': 4.399599733155437e-05,\n",
       "  'epoch': 0.6,\n",
       "  'step': 9000},\n",
       " {'loss': 0.0399,\n",
       "  'learning_rate': 4.366244162775184e-05,\n",
       "  'epoch': 0.63,\n",
       "  'step': 9500},\n",
       " {'loss': 0.0372,\n",
       "  'learning_rate': 4.33288859239493e-05,\n",
       "  'epoch': 0.67,\n",
       "  'step': 10000},\n",
       " {'loss': 0.0359,\n",
       "  'learning_rate': 4.299533022014677e-05,\n",
       "  'epoch': 0.7,\n",
       "  'step': 10500},\n",
       " {'loss': 0.0349,\n",
       "  'learning_rate': 4.266177451634423e-05,\n",
       "  'epoch': 0.73,\n",
       "  'step': 11000},\n",
       " {'loss': 0.0372,\n",
       "  'learning_rate': 4.23282188125417e-05,\n",
       "  'epoch': 0.77,\n",
       "  'step': 11500},\n",
       " {'loss': 0.0318,\n",
       "  'learning_rate': 4.199466310873916e-05,\n",
       "  'epoch': 0.8,\n",
       "  'step': 12000},\n",
       " {'loss': 0.0332,\n",
       "  'learning_rate': 4.166110740493663e-05,\n",
       "  'epoch': 0.83,\n",
       "  'step': 12500},\n",
       " {'loss': 0.0331,\n",
       "  'learning_rate': 4.132755170113409e-05,\n",
       "  'epoch': 0.87,\n",
       "  'step': 13000},\n",
       " {'loss': 0.0301,\n",
       "  'learning_rate': 4.099399599733156e-05,\n",
       "  'epoch': 0.9,\n",
       "  'step': 13500},\n",
       " {'loss': 0.0321,\n",
       "  'learning_rate': 4.0660440293529026e-05,\n",
       "  'epoch': 0.93,\n",
       "  'step': 14000},\n",
       " {'loss': 0.033,\n",
       "  'learning_rate': 4.032688458972649e-05,\n",
       "  'epoch': 0.97,\n",
       "  'step': 14500},\n",
       " {'eval_loss': 0.02946559153497219,\n",
       "  'eval_accuracy': 0.9912439619821168,\n",
       "  'eval_f1-score': 0.9911871940646529,\n",
       "  'eval_runtime': 2743.4628,\n",
       "  'eval_samples_per_second': 174.841,\n",
       "  'eval_steps_per_second': 0.683,\n",
       "  'epoch': 1.0,\n",
       "  'step': 14990},\n",
       " {'loss': 0.03,\n",
       "  'learning_rate': 3.9993328885923955e-05,\n",
       "  'epoch': 1.0,\n",
       "  'step': 15000},\n",
       " {'loss': 0.0286,\n",
       "  'learning_rate': 3.965977318212142e-05,\n",
       "  'epoch': 1.03,\n",
       "  'step': 15500},\n",
       " {'loss': 0.0285,\n",
       "  'learning_rate': 3.9326217478318885e-05,\n",
       "  'epoch': 1.07,\n",
       "  'step': 16000},\n",
       " {'loss': 0.0277,\n",
       "  'learning_rate': 3.8992661774516346e-05,\n",
       "  'epoch': 1.1,\n",
       "  'step': 16500},\n",
       " {'loss': 0.0262,\n",
       "  'learning_rate': 3.8659106070713814e-05,\n",
       "  'epoch': 1.13,\n",
       "  'step': 17000},\n",
       " {'loss': 0.0281,\n",
       "  'learning_rate': 3.8325550366911275e-05,\n",
       "  'epoch': 1.17,\n",
       "  'step': 17500},\n",
       " {'loss': 0.0269,\n",
       "  'learning_rate': 3.799199466310874e-05,\n",
       "  'epoch': 1.2,\n",
       "  'step': 18000},\n",
       " {'loss': 0.025,\n",
       "  'learning_rate': 3.765843895930621e-05,\n",
       "  'epoch': 1.23,\n",
       "  'step': 18500},\n",
       " {'loss': 0.0249,\n",
       "  'learning_rate': 3.732488325550367e-05,\n",
       "  'epoch': 1.27,\n",
       "  'step': 19000},\n",
       " {'loss': 0.0232,\n",
       "  'learning_rate': 3.699132755170114e-05,\n",
       "  'epoch': 1.3,\n",
       "  'step': 19500},\n",
       " {'loss': 0.0269,\n",
       "  'learning_rate': 3.66577718478986e-05,\n",
       "  'epoch': 1.33,\n",
       "  'step': 20000},\n",
       " {'loss': 0.0247,\n",
       "  'learning_rate': 3.632421614409607e-05,\n",
       "  'epoch': 1.37,\n",
       "  'step': 20500},\n",
       " {'loss': 0.0243,\n",
       "  'learning_rate': 3.599066044029353e-05,\n",
       "  'epoch': 1.4,\n",
       "  'step': 21000},\n",
       " {'loss': 0.0245,\n",
       "  'learning_rate': 3.5657104736491e-05,\n",
       "  'epoch': 1.43,\n",
       "  'step': 21500},\n",
       " {'loss': 0.0227,\n",
       "  'learning_rate': 3.532354903268846e-05,\n",
       "  'epoch': 1.47,\n",
       "  'step': 22000},\n",
       " {'loss': 0.0242,\n",
       "  'learning_rate': 3.498999332888593e-05,\n",
       "  'epoch': 1.5,\n",
       "  'step': 22500},\n",
       " {'loss': 0.0235,\n",
       "  'learning_rate': 3.46564376250834e-05,\n",
       "  'epoch': 1.53,\n",
       "  'step': 23000},\n",
       " {'loss': 0.021,\n",
       "  'learning_rate': 3.432288192128085e-05,\n",
       "  'epoch': 1.57,\n",
       "  'step': 23500},\n",
       " {'loss': 0.0211,\n",
       "  'learning_rate': 3.398932621747832e-05,\n",
       "  'epoch': 1.6,\n",
       "  'step': 24000},\n",
       " {'loss': 0.0205,\n",
       "  'learning_rate': 3.365577051367578e-05,\n",
       "  'epoch': 1.63,\n",
       "  'step': 24500},\n",
       " {'loss': 0.0204,\n",
       "  'learning_rate': 3.332221480987325e-05,\n",
       "  'epoch': 1.67,\n",
       "  'step': 25000},\n",
       " {'loss': 0.0203,\n",
       "  'learning_rate': 3.298865910607071e-05,\n",
       "  'epoch': 1.7,\n",
       "  'step': 25500},\n",
       " {'loss': 0.0225,\n",
       "  'learning_rate': 3.265510340226818e-05,\n",
       "  'epoch': 1.73,\n",
       "  'step': 26000},\n",
       " {'loss': 0.0193,\n",
       "  'learning_rate': 3.2321547698465646e-05,\n",
       "  'epoch': 1.77,\n",
       "  'step': 26500},\n",
       " {'loss': 0.0212,\n",
       "  'learning_rate': 3.198799199466311e-05,\n",
       "  'epoch': 1.8,\n",
       "  'step': 27000},\n",
       " {'loss': 0.0204,\n",
       "  'learning_rate': 3.1654436290860575e-05,\n",
       "  'epoch': 1.83,\n",
       "  'step': 27500},\n",
       " {'loss': 0.0204,\n",
       "  'learning_rate': 3.1320880587058036e-05,\n",
       "  'epoch': 1.87,\n",
       "  'step': 28000},\n",
       " {'loss': 0.0181,\n",
       "  'learning_rate': 3.0987324883255504e-05,\n",
       "  'epoch': 1.9,\n",
       "  'step': 28500},\n",
       " {'loss': 0.0208,\n",
       "  'learning_rate': 3.0653769179452966e-05,\n",
       "  'epoch': 1.93,\n",
       "  'step': 29000},\n",
       " {'loss': 0.0186,\n",
       "  'learning_rate': 3.0320213475650434e-05,\n",
       "  'epoch': 1.97,\n",
       "  'step': 29500},\n",
       " {'eval_loss': 0.017275430262088776,\n",
       "  'eval_accuracy': 0.9951758400063377,\n",
       "  'eval_f1-score': 0.9951489226503366,\n",
       "  'eval_runtime': 3249.2702,\n",
       "  'eval_samples_per_second': 147.624,\n",
       "  'eval_steps_per_second': 0.577,\n",
       "  'epoch': 2.0,\n",
       "  'step': 29980},\n",
       " {'loss': 0.0174,\n",
       "  'learning_rate': 2.99866577718479e-05,\n",
       "  'epoch': 2.0,\n",
       "  'step': 30000},\n",
       " {'loss': 0.0164,\n",
       "  'learning_rate': 2.9653102068045363e-05,\n",
       "  'epoch': 2.03,\n",
       "  'step': 30500},\n",
       " {'loss': 0.0187,\n",
       "  'learning_rate': 2.9319546364242828e-05,\n",
       "  'epoch': 2.07,\n",
       "  'step': 31000},\n",
       " {'loss': 0.0172,\n",
       "  'learning_rate': 2.8985990660440292e-05,\n",
       "  'epoch': 2.1,\n",
       "  'step': 31500},\n",
       " {'loss': 0.0168,\n",
       "  'learning_rate': 2.8652434956637757e-05,\n",
       "  'epoch': 2.13,\n",
       "  'step': 32000},\n",
       " {'loss': 0.0173,\n",
       "  'learning_rate': 2.831887925283522e-05,\n",
       "  'epoch': 2.17,\n",
       "  'step': 32500},\n",
       " {'loss': 0.0168,\n",
       "  'learning_rate': 2.798532354903269e-05,\n",
       "  'epoch': 2.2,\n",
       "  'step': 33000},\n",
       " {'loss': 0.0162,\n",
       "  'learning_rate': 2.7651767845230154e-05,\n",
       "  'epoch': 2.23,\n",
       "  'step': 33500},\n",
       " {'loss': 0.0165,\n",
       "  'learning_rate': 2.731821214142762e-05,\n",
       "  'epoch': 2.27,\n",
       "  'step': 34000},\n",
       " {'loss': 0.0152,\n",
       "  'learning_rate': 2.6984656437625084e-05,\n",
       "  'epoch': 2.3,\n",
       "  'step': 34500},\n",
       " {'loss': 0.0162,\n",
       "  'learning_rate': 2.6651100733822548e-05,\n",
       "  'epoch': 2.33,\n",
       "  'step': 35000},\n",
       " {'loss': 0.0144,\n",
       "  'learning_rate': 2.6317545030020013e-05,\n",
       "  'epoch': 2.37,\n",
       "  'step': 35500},\n",
       " {'loss': 0.0145,\n",
       "  'learning_rate': 2.5983989326217478e-05,\n",
       "  'epoch': 2.4,\n",
       "  'step': 36000},\n",
       " {'loss': 0.0157,\n",
       "  'learning_rate': 2.5650433622414942e-05,\n",
       "  'epoch': 2.43,\n",
       "  'step': 36500},\n",
       " {'loss': 0.0163,\n",
       "  'learning_rate': 2.5316877918612407e-05,\n",
       "  'epoch': 2.47,\n",
       "  'step': 37000},\n",
       " {'loss': 0.0141,\n",
       "  'learning_rate': 2.4983322214809875e-05,\n",
       "  'epoch': 2.5,\n",
       "  'step': 37500},\n",
       " {'loss': 0.0133,\n",
       "  'learning_rate': 2.464976651100734e-05,\n",
       "  'epoch': 2.54,\n",
       "  'step': 38000},\n",
       " {'loss': 0.0144,\n",
       "  'learning_rate': 2.4316210807204804e-05,\n",
       "  'epoch': 2.57,\n",
       "  'step': 38500},\n",
       " {'loss': 0.0136,\n",
       "  'learning_rate': 2.398265510340227e-05,\n",
       "  'epoch': 2.6,\n",
       "  'step': 39000},\n",
       " {'loss': 0.015,\n",
       "  'learning_rate': 2.3649099399599734e-05,\n",
       "  'epoch': 2.64,\n",
       "  'step': 39500},\n",
       " {'loss': 0.0147,\n",
       "  'learning_rate': 2.3315543695797198e-05,\n",
       "  'epoch': 2.67,\n",
       "  'step': 40000},\n",
       " {'loss': 0.015,\n",
       "  'learning_rate': 2.2981987991994663e-05,\n",
       "  'epoch': 2.7,\n",
       "  'step': 40500},\n",
       " {'loss': 0.0155,\n",
       "  'learning_rate': 2.2648432288192127e-05,\n",
       "  'epoch': 2.74,\n",
       "  'step': 41000},\n",
       " {'loss': 0.0145,\n",
       "  'learning_rate': 2.2314876584389592e-05,\n",
       "  'epoch': 2.77,\n",
       "  'step': 41500},\n",
       " {'loss': 0.0139,\n",
       "  'learning_rate': 2.198132088058706e-05,\n",
       "  'epoch': 2.8,\n",
       "  'step': 42000},\n",
       " {'loss': 0.0132,\n",
       "  'learning_rate': 2.1647765176784525e-05,\n",
       "  'epoch': 2.84,\n",
       "  'step': 42500},\n",
       " {'loss': 0.0135,\n",
       "  'learning_rate': 2.131420947298199e-05,\n",
       "  'epoch': 2.87,\n",
       "  'step': 43000},\n",
       " {'loss': 0.0139,\n",
       "  'learning_rate': 2.0980653769179454e-05,\n",
       "  'epoch': 2.9,\n",
       "  'step': 43500},\n",
       " {'loss': 0.0145,\n",
       "  'learning_rate': 2.064709806537692e-05,\n",
       "  'epoch': 2.94,\n",
       "  'step': 44000},\n",
       " {'loss': 0.0126,\n",
       "  'learning_rate': 2.0313542361574383e-05,\n",
       "  'epoch': 2.97,\n",
       "  'step': 44500},\n",
       " {'eval_loss': 0.012999790720641613,\n",
       "  'eval_accuracy': 0.9964308721222342,\n",
       "  'eval_f1-score': 0.9964104165906558,\n",
       "  'eval_runtime': 2952.5613,\n",
       "  'eval_samples_per_second': 162.459,\n",
       "  'eval_steps_per_second': 0.635,\n",
       "  'epoch': 3.0,\n",
       "  'step': 44970},\n",
       " {'loss': 0.0141,\n",
       "  'learning_rate': 1.9979986657771848e-05,\n",
       "  'epoch': 3.0,\n",
       "  'step': 45000},\n",
       " {'loss': 0.0105,\n",
       "  'learning_rate': 1.9646430953969313e-05,\n",
       "  'epoch': 3.04,\n",
       "  'step': 45500},\n",
       " {'loss': 0.011,\n",
       "  'learning_rate': 1.9312875250166777e-05,\n",
       "  'epoch': 3.07,\n",
       "  'step': 46000},\n",
       " {'loss': 0.0107,\n",
       "  'learning_rate': 1.8979319546364242e-05,\n",
       "  'epoch': 3.1,\n",
       "  'step': 46500},\n",
       " {'loss': 0.0118,\n",
       "  'learning_rate': 1.864576384256171e-05,\n",
       "  'epoch': 3.14,\n",
       "  'step': 47000},\n",
       " {'loss': 0.0107,\n",
       "  'learning_rate': 1.8312208138759175e-05,\n",
       "  'epoch': 3.17,\n",
       "  'step': 47500},\n",
       " {'loss': 0.0118,\n",
       "  'learning_rate': 1.797865243495664e-05,\n",
       "  'epoch': 3.2,\n",
       "  'step': 48000},\n",
       " {'loss': 0.011,\n",
       "  'learning_rate': 1.7645096731154104e-05,\n",
       "  'epoch': 3.24,\n",
       "  'step': 48500},\n",
       " {'loss': 0.0119,\n",
       "  'learning_rate': 1.731154102735157e-05,\n",
       "  'epoch': 3.27,\n",
       "  'step': 49000},\n",
       " {'loss': 0.011,\n",
       "  'learning_rate': 1.6977985323549033e-05,\n",
       "  'epoch': 3.3,\n",
       "  'step': 49500},\n",
       " {'loss': 0.0118,\n",
       "  'learning_rate': 1.6644429619746498e-05,\n",
       "  'epoch': 3.34,\n",
       "  'step': 50000},\n",
       " {'loss': 0.0114,\n",
       "  'learning_rate': 1.6310873915943963e-05,\n",
       "  'epoch': 3.37,\n",
       "  'step': 50500},\n",
       " {'loss': 0.0103,\n",
       "  'learning_rate': 1.5977318212141427e-05,\n",
       "  'epoch': 3.4,\n",
       "  'step': 51000},\n",
       " {'loss': 0.0106,\n",
       "  'learning_rate': 1.5643762508338895e-05,\n",
       "  'epoch': 3.44,\n",
       "  'step': 51500},\n",
       " {'loss': 0.0121,\n",
       "  'learning_rate': 1.531020680453636e-05,\n",
       "  'epoch': 3.47,\n",
       "  'step': 52000},\n",
       " {'loss': 0.0112,\n",
       "  'learning_rate': 1.4976651100733825e-05,\n",
       "  'epoch': 3.5,\n",
       "  'step': 52500},\n",
       " {'loss': 0.0103,\n",
       "  'learning_rate': 1.464309539693129e-05,\n",
       "  'epoch': 3.54,\n",
       "  'step': 53000},\n",
       " {'loss': 0.0106,\n",
       "  'learning_rate': 1.4309539693128754e-05,\n",
       "  'epoch': 3.57,\n",
       "  'step': 53500},\n",
       " {'loss': 0.0108,\n",
       "  'learning_rate': 1.3975983989326219e-05,\n",
       "  'epoch': 3.6,\n",
       "  'step': 54000},\n",
       " {'loss': 0.0093,\n",
       "  'learning_rate': 1.3642428285523683e-05,\n",
       "  'epoch': 3.64,\n",
       "  'step': 54500},\n",
       " {'loss': 0.0113,\n",
       "  'learning_rate': 1.330887258172115e-05,\n",
       "  'epoch': 3.67,\n",
       "  'step': 55000},\n",
       " {'loss': 0.0111,\n",
       "  'learning_rate': 1.2975316877918614e-05,\n",
       "  'epoch': 3.7,\n",
       "  'step': 55500},\n",
       " {'loss': 0.0107,\n",
       "  'learning_rate': 1.2641761174116079e-05,\n",
       "  'epoch': 3.74,\n",
       "  'step': 56000},\n",
       " {'loss': 0.01,\n",
       "  'learning_rate': 1.2308205470313544e-05,\n",
       "  'epoch': 3.77,\n",
       "  'step': 56500},\n",
       " {'loss': 0.0114,\n",
       "  'learning_rate': 1.1974649766511008e-05,\n",
       "  'epoch': 3.8,\n",
       "  'step': 57000},\n",
       " {'loss': 0.0102,\n",
       "  'learning_rate': 1.1641094062708473e-05,\n",
       "  'epoch': 3.84,\n",
       "  'step': 57500},\n",
       " {'loss': 0.0098,\n",
       "  'learning_rate': 1.1307538358905937e-05,\n",
       "  'epoch': 3.87,\n",
       "  'step': 58000},\n",
       " {'loss': 0.0102,\n",
       "  'learning_rate': 1.0973982655103402e-05,\n",
       "  'epoch': 3.9,\n",
       "  'step': 58500},\n",
       " {'loss': 0.0094,\n",
       "  'learning_rate': 1.0640426951300868e-05,\n",
       "  'epoch': 3.94,\n",
       "  'step': 59000},\n",
       " {'loss': 0.01,\n",
       "  'learning_rate': 1.0306871247498333e-05,\n",
       "  'epoch': 3.97,\n",
       "  'step': 59500},\n",
       " {'eval_loss': 0.010789062827825546,\n",
       "  'eval_accuracy': 0.9970333709287029,\n",
       "  'eval_f1-score': 0.9970173798528738,\n",
       "  'eval_runtime': 2853.7502,\n",
       "  'eval_samples_per_second': 168.084,\n",
       "  'eval_steps_per_second': 0.657,\n",
       "  'epoch': 4.0,\n",
       "  'step': 59960},\n",
       " {'loss': 0.0098,\n",
       "  'learning_rate': 9.973315543695798e-06,\n",
       "  'epoch': 4.0,\n",
       "  'step': 60000},\n",
       " {'loss': 0.0091,\n",
       "  'learning_rate': 9.639759839893262e-06,\n",
       "  'epoch': 4.04,\n",
       "  'step': 60500},\n",
       " {'loss': 0.0092,\n",
       "  'learning_rate': 9.306204136090727e-06,\n",
       "  'epoch': 4.07,\n",
       "  'step': 61000},\n",
       " {'loss': 0.0078,\n",
       "  'learning_rate': 8.972648432288193e-06,\n",
       "  'epoch': 4.1,\n",
       "  'step': 61500},\n",
       " {'loss': 0.0086,\n",
       "  'learning_rate': 8.639092728485658e-06,\n",
       "  'epoch': 4.14,\n",
       "  'step': 62000},\n",
       " {'loss': 0.0081,\n",
       "  'learning_rate': 8.305537024683123e-06,\n",
       "  'epoch': 4.17,\n",
       "  'step': 62500},\n",
       " {'loss': 0.009,\n",
       "  'learning_rate': 7.971981320880587e-06,\n",
       "  'epoch': 4.2,\n",
       "  'step': 63000},\n",
       " {'loss': 0.0089,\n",
       "  'learning_rate': 7.638425617078054e-06,\n",
       "  'epoch': 4.24,\n",
       "  'step': 63500},\n",
       " {'loss': 0.0083,\n",
       "  'learning_rate': 7.304869913275518e-06,\n",
       "  'epoch': 4.27,\n",
       "  'step': 64000},\n",
       " {'loss': 0.0076,\n",
       "  'learning_rate': 6.971314209472981e-06,\n",
       "  'epoch': 4.3,\n",
       "  'step': 64500},\n",
       " {'loss': 0.0091,\n",
       "  'learning_rate': 6.637758505670447e-06,\n",
       "  'epoch': 4.34,\n",
       "  'step': 65000},\n",
       " {'loss': 0.008,\n",
       "  'learning_rate': 6.3042028018679115e-06,\n",
       "  'epoch': 4.37,\n",
       "  'step': 65500},\n",
       " {'loss': 0.0088,\n",
       "  'learning_rate': 5.970647098065377e-06,\n",
       "  'epoch': 4.4,\n",
       "  'step': 66000},\n",
       " {'loss': 0.0085,\n",
       "  'learning_rate': 5.6370913942628425e-06,\n",
       "  'epoch': 4.44,\n",
       "  'step': 66500},\n",
       " {'loss': 0.0083,\n",
       "  'learning_rate': 5.303535690460307e-06,\n",
       "  'epoch': 4.47,\n",
       "  'step': 67000},\n",
       " {'loss': 0.0076,\n",
       "  'learning_rate': 4.969979986657772e-06,\n",
       "  'epoch': 4.5,\n",
       "  'step': 67500},\n",
       " {'loss': 0.0079,\n",
       "  'learning_rate': 4.6364242828552364e-06,\n",
       "  'epoch': 4.54,\n",
       "  'step': 68000},\n",
       " {'loss': 0.0091,\n",
       "  'learning_rate': 4.302868579052702e-06,\n",
       "  'epoch': 4.57,\n",
       "  'step': 68500},\n",
       " {'loss': 0.008,\n",
       "  'learning_rate': 3.969312875250167e-06,\n",
       "  'epoch': 4.6,\n",
       "  'step': 69000},\n",
       " {'loss': 0.0087,\n",
       "  'learning_rate': 3.6357571714476317e-06,\n",
       "  'epoch': 4.64,\n",
       "  'step': 69500},\n",
       " {'loss': 0.0083,\n",
       "  'learning_rate': 3.3022014676450967e-06,\n",
       "  'epoch': 4.67,\n",
       "  'step': 70000},\n",
       " {'loss': 0.0088,\n",
       "  'learning_rate': 2.968645763842562e-06,\n",
       "  'epoch': 4.7,\n",
       "  'step': 70500},\n",
       " {'loss': 0.0081,\n",
       "  'learning_rate': 2.635090060040027e-06,\n",
       "  'epoch': 4.74,\n",
       "  'step': 71000},\n",
       " {'loss': 0.0076,\n",
       "  'learning_rate': 2.301534356237492e-06,\n",
       "  'epoch': 4.77,\n",
       "  'step': 71500},\n",
       " {'loss': 0.0073,\n",
       "  'learning_rate': 1.9679786524349566e-06,\n",
       "  'epoch': 4.8,\n",
       "  'step': 72000},\n",
       " {'loss': 0.0085,\n",
       "  'learning_rate': 1.6344229486324217e-06,\n",
       "  'epoch': 4.84,\n",
       "  'step': 72500},\n",
       " {'loss': 0.0087,\n",
       "  'learning_rate': 1.3008672448298865e-06,\n",
       "  'epoch': 4.87,\n",
       "  'step': 73000},\n",
       " {'loss': 0.0082,\n",
       "  'learning_rate': 9.673115410273516e-07,\n",
       "  'epoch': 4.9,\n",
       "  'step': 73500},\n",
       " {'loss': 0.0078,\n",
       "  'learning_rate': 6.337558372248166e-07,\n",
       "  'epoch': 4.94,\n",
       "  'step': 74000},\n",
       " {'loss': 0.008,\n",
       "  'learning_rate': 3.002001334222815e-07,\n",
       "  'epoch': 4.97,\n",
       "  'step': 74500},\n",
       " {'eval_loss': 0.010324587114155293,\n",
       "  'eval_accuracy': 0.997304391153066,\n",
       "  'eval_f1-score': 0.9972906032914309,\n",
       "  'eval_runtime': 2200.6035,\n",
       "  'eval_samples_per_second': 217.972,\n",
       "  'eval_steps_per_second': 0.852,\n",
       "  'epoch': 5.0,\n",
       "  'step': 74950},\n",
       " {'train_runtime': 177793.8878,\n",
       "  'train_samples_per_second': 53.958,\n",
       "  'train_steps_per_second': 0.422,\n",
       "  'total_flos': 2.524621608520704e+18,\n",
       "  'train_loss': 0.026328859431016757,\n",
       "  'epoch': 5.0,\n",
       "  'step': 74950}]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.state.log_history"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5aea65a-5e34-489c-a735-2e8327508914",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "81e457b6-ee65-4a11-bf55-c0e42e44244f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e17b1efe-09e2-4b64-9464-b706c3759dbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./DATA_TEST.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4b39e658-8c27-4d7f-9293-6099190357b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>packet_dat</th>\n",
       "      <th>attack_cat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>75 75 864 -1 80 48846 2948 2896 64 0 8 0 -1 32...</td>\n",
       "      <td>DoS Hulk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0 0 2 -1 80 56471 10175 10135 64 0 5 3 -1 112 ...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>500000 0 0 -1 80 46678 1500 1448 64 0 8 0 -1 7...</td>\n",
       "      <td>DoS Hulk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>240 481 956354 -1 80 33554 1500 1448 64 0 8 0 ...</td>\n",
       "      <td>DoS Hulk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>111 16 147912 -1 38156 25 1500 1460 254 0 5 0 ...</td>\n",
       "      <td>DoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509045</th>\n",
       "      <td>30 40 116520 -1 80 56105 5795 5755 64 0 5 2 -1...</td>\n",
       "      <td>DDoS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509046</th>\n",
       "      <td>666666 0 0 -1 54614 80 422 370 62 0 8 3 -1 71 ...</td>\n",
       "      <td>DoS Hulk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509047</th>\n",
       "      <td>500000 0 0 -1 41110 80 428 376 62 0 8 3 -1 71 ...</td>\n",
       "      <td>DoS Hulk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509048</th>\n",
       "      <td>77 12 103767 -1 49888 25 1500 1460 254 0 5 0 -...</td>\n",
       "      <td>Generic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509049</th>\n",
       "      <td>0 0 124 -1 80 51136 5844 5792 64 0 8 0 -1 120 ...</td>\n",
       "      <td>DoS Hulk</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>509050 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               packet_dat attack_cat\n",
       "0       75 75 864 -1 80 48846 2948 2896 64 0 8 0 -1 32...   DoS Hulk\n",
       "1       0 0 2 -1 80 56471 10175 10135 64 0 5 3 -1 112 ...       DDoS\n",
       "2       500000 0 0 -1 80 46678 1500 1448 64 0 8 0 -1 7...   DoS Hulk\n",
       "3       240 481 956354 -1 80 33554 1500 1448 64 0 8 0 ...   DoS Hulk\n",
       "4       111 16 147912 -1 38156 25 1500 1460 254 0 5 0 ...        DoS\n",
       "...                                                   ...        ...\n",
       "509045  30 40 116520 -1 80 56105 5795 5755 64 0 5 2 -1...       DDoS\n",
       "509046  666666 0 0 -1 54614 80 422 370 62 0 8 3 -1 71 ...   DoS Hulk\n",
       "509047  500000 0 0 -1 41110 80 428 376 62 0 8 3 -1 71 ...   DoS Hulk\n",
       "509048  77 12 103767 -1 49888 25 1500 1460 254 0 5 0 -...    Generic\n",
       "509049  0 0 124 -1 80 51136 5844 5792 64 0 8 0 -1 120 ...   DoS Hulk\n",
       "\n",
       "[509050 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "b82b573c-ef29-4070-96ef-e4bcb477868a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DoS Hulk                      178975\n",
       "Normal                         77908\n",
       "DDoS                           54891\n",
       "Heartbleed                     50304\n",
       "SSH Patator                    25492\n",
       "DoS                            21192\n",
       "Exploits                       19620\n",
       "Generic                        14777\n",
       "FTP Patator                    13505\n",
       "Fuzzers                        10789\n",
       "Reconnaissance                 10395\n",
       "DoS GoldenEye                  10131\n",
       "Web Attack - Brute Force        5616\n",
       "Infiltration                    3260\n",
       "Worms                           2914\n",
       "DoS SlowHTTPTest                2432\n",
       "DoS Slowloris                   1852\n",
       "Web Attack - XSS                1549\n",
       "Bot                             1460\n",
       "Analysis                         793\n",
       "Backdoor                         507\n",
       "Shellcode                        435\n",
       "Port Scan                        238\n",
       "Web Attack - SQL Injection        15\n",
       "Name: attack_cat, dtype: int64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.attack_cat.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c239eb89-7498-459f-ba1a-66568e491b56",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ae2d9d97-87f3-45bc-ad98-e553d9b66351",
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe = pipeline(\"text-classification\", model=\"rdpahalavan/bert-network-packet-flow-header-payload\", device=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "531ecda2-e270-4305-bc51-f38924ff9d27",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = pipe(df['packet_dat'].to_list(), truncation=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2a4ce7a8-7bb4-4236-b6f0-e156346da56a",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = [pred['label'] for pred in predictions]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "305715ad-790d-45da-a60c-818db0867db7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "38d24f93-e92f-497c-9986-872697bc5b82",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9948453000687555"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(df['attack_cat'], pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "3d31f4f7-e6fe-4c0e-807a-4b4fa75f7f45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9948453000687555"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(df['attack_cat'], pred, average='micro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "258526a8-95f8-4e2e-8afa-5f9a55a499b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9846018986149065"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(df['attack_cat'], pred, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18424ec1-9cf1-4f92-9da8-2b01de650430",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "15ac128c-60c0-4767-9984-2b91d52ffb6e",
   "metadata": {},
   "source": [
    "# Fine-tuning Text Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "02d56239-bcad-431c-839b-0617ab907deb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a8bcb0d2-4fa7-4553-b344-e55843179bd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from datasets import Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ccef2f01-4178-49e5-bf43-52f862676bdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_csv('./Text Generation/QUESTION.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8e8e5777-581d-4f9f-a37e-11dea7b96b72",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_csv('./Text Generation/ANSWER.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2e459647-4046-4251-9872-47b2bc4abcb3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please run\n",
      "\n",
      "python -m bitsandbytes\n",
      "\n",
      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "bin /data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32\n",
      "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.0\n",
      "CUDA SETUP: Detected CUDA version 117\n",
      "CUDA SETUP: Loading binary /data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable.\n",
      "  warn(\"The installed version of bitsandbytes was compiled without GPU support. \"\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/user/rajkumardheivanayahi/oauth_callback')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('[\"access'), PosixPath('servers!server=rajkumardheivanayahi/\", \"access'), PosixPath('servers!user=rajkumardheivanayahi\"]')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('http'), PosixPath('8081/hub/api'), PosixPath('//127.0.0.1')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('http'), PosixPath('8081/hub/api/users/rajkumardheivanayahi/activity'), PosixPath('//127.0.0.1')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/user/rajkumardheivanayahi')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('49529/user/rajkumardheivanayahi'), PosixPath('http'), PosixPath('//127.0.0.1')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//matplotlib_inline.backend_inline')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/cuda/lib64')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: No libcudart.so found! Install CUDA or the cudatoolkit package (anaconda)!\n",
      "  warn(msg)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import GPT2Tokenizer, GPT2LMHeadModel, DataCollatorForLanguageModeling, AutoTokenizer, AutoModelForCausalLM\n",
    "from transformers import Trainer, TrainingArguments\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "model_name_or_path = \"gpt2\"\n",
    "tokenizer = GPT2Tokenizer.from_pretrained(model_name_or_path)\n",
    "model = GPT2LMHeadModel.from_pretrained(model_name_or_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e3a4ff11-58e2-4101-a062-876cc5db00cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.pad_token = tokenizer.eos_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c8371c1f-da1b-4179-8d23-398ce0e90caa",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1['text'] = df1['packet+payload'] + tokenizer.eos_token + df2['explanation'] + tokenizer.eos_token\n",
    "df1['text'].to_frame().to_csv('text_data.csv', index=False)\n",
    "\n",
    "dataset = Dataset.from_pandas(df1[['text']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7fa79c0a-7b99-4fc5-a2f7-d1e2e52ff7c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text'],\n",
       "    num_rows: 53611\n",
       "})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4bf58aca-cf1f-43fc-8c13-121ebd01f5c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': 'IP version: 4.0. IP ihl: 5.0. IP tos: 0x0. IP len: 219.0. IP frag: 0.0. IP ttl: 62.0. IP proto: tcp. IP src: 175.45.176.2. IP dst: 149.171.126.19. TCP sport: 19661. TCP dport: http. TCP seq: 1739521222.0. TCP ack: 578738309.0. TCP dataofs: 5.0. TCP reserved: 0.0. TCP flags: PA. TCP window: 16383.0. TCP chksum: 0x6ede. TCP urgptr: 0.0. TCP options: []. The payload contains these words: GET  GcI.asp HTTP Host  OpLqwnfYOUser Agent  Mozilla compatible  MSIE   Windows NT   Trident Accept Connection  keep aliveContent Length<|endoftext|>This network packet is an IPv4 packet with a header length of 5 and a type of service value of 0x0. The total length of the packet is 219 bytes, with no fragmentation information. The time to live is set to 62, and the protocol used in the packet is TCP. The source IP address is 175.45.176.2, and the destination IP address is 149.171.126.19. In the TCP header, the source port is 19661, and the destination port is for HTTP. The sequence number is 1739521222, and the acknowledgement number is 578738309. The data offset is 5, and there are no reserved bits. The TCP flags indicate that it is a push and acknowledgement packet. The window size is 16383 bytes, and the checksum is 0x6ede. The payload contains words like \"GET,\" \"HTTP,\" \"Host,\" \"User Agent,\" \"Mozilla compatible,\" \"MSIE,\" \"Windows NT,\" \"Trident,\" \"Accept,\" \"Connection,\" \"keep alive,\" and \"Content Length.\" Overall, there don\\'t seem to be any abnormalities in the packet.<|endoftext|>'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "702769d7-7784-4eb8-bba4-5cbe905df1bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/53611 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def tokenize_function(examples):\n",
    "    return tokenizer(examples[\"text\"], truncation=True)\n",
    "\n",
    "tokenized_dataset = dataset.map(tokenize_function, batched=True, remove_columns=[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0e26154a-8440-4d5d-8bd0-62106c47f1a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text', 'input_ids', 'attention_mask'],\n",
       "    num_rows: 53611\n",
       "})"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a4bbf6c3-9e8d-456c-be7d-315c450654aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = DataCollatorForLanguageModeling(\n",
    "    tokenizer=tokenizer, mlm=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "84209e41-d73d-4fb2-bba7-64a0aad5211b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n"
     ]
    }
   ],
   "source": [
    "# Initialize the Trainer with TrainingArguments and GPT2LMHeadModel\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./falcon_finetuned\", #The output directory\n",
    "    overwrite_output_dir=True, #overwrite the content of the output directory\n",
    "    num_train_epochs=1, # number of training epochs\n",
    "    per_device_train_batch_size=16, # batch size for training\n",
    "    save_steps=10_000, # after # steps model is saved \n",
    "    save_total_limit=2, # the total amount of checkpoints. Deletes the older checkpoints.\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "656a1430-09cc-42ce-b005-f6fc85609e72",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    data_collator=data_collator,\n",
    "    train_dataset=tokenized_dataset\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f03da4f3-a19f-4d78-912e-9d5ae4a8d342",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1676' max='1676' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1676/1676 53:33, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>1.270600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>1.071800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>1.031000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1676, training_loss=1.112808500667745, metrics={'train_runtime': 3218.6708, 'train_samples_per_second': 16.656, 'train_steps_per_second': 0.521, 'total_flos': 2.7945542098944e+16, 'train_loss': 1.112808500667745, 'epoch': 1.0})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "33776fe5-dca3-4c8e-a57a-b265fabb69ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.save_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ad4bec78-3f16-4fee-858e-e745668f3b2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import GPT2LMHeadModel, GPT2Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9fb7685d-04c8-43b3-8490-8e3ad1f5c2fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = GPT2LMHeadModel.from_pretrained(\"falcon_finetuned\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0e764ea9-3a53-43f2-97f2-a8584cf2bcab",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cdf445d1-b38d-4850-bb68-b42adee3fd9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded_prompt = tokenizer.encode(df1['packet+payload'][1] + tokenizer.eos_token, add_special_tokens=False, return_tensors=\"pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "684a56e4-cebf-476f-8efb-bad69001f138",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
      "A decoder-only architecture is being used, but right-padding was detected! For correct generation results, please set `padding_side='left'` when initializing the tokenizer.\n"
     ]
    }
   ],
   "source": [
    "output_sequences = model.generate(\n",
    "        input_ids=encoded_prompt,\n",
    "        max_new_tokens=100,\n",
    "        temperature=1.0,\n",
    "        top_k=0,\n",
    "        top_p=0.9,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b5212af2-9836-4170-b398-fa772710fb4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'IP version: 4.0. IP ihl: 5.0. IP tos: 0x0. IP len: 410.0. IP frag: 0.0. IP ttl: 252.0. IP proto: tcp. IP src: 149.171.126.19. IP dst: 175.45.176.3. TCP sport: http. TCP dport: 25542. TCP seq: 1495070304.0. TCP ack: 1226692451.0. TCP dataofs: 5.0. TCP reserved: 0.0. TCP flags: A. TCP window: 16383.0. TCP chksum: 0x3029. TCP urgptr: 0.0. TCP options: []. The payload contains these words: HTTP  OKDate  Tue  Feb   GMTServer  Apache Host  voJPszioCGFZFnWmGOgmnmJLAULyLast Modified  Tue  Feb   GMTAccept Ranges  bytesContent Length Connection  closeContent Type  text html html objectclassid clsid object script src script.js script<|endoftext|>This network packet is an IPv4 packet with a header length (ihl) of 5 and a type of service (tos) value of 0x0. The total length of the packet is 410 bytes. The packet is not fragmented (frag: 0.0) and has a time to live (ttl) value of 252. The protocol used is TCP. The source IP address is 149.171.126.19 and the destination IP address is 175.45.176.'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_sequence = output_sequences[0].tolist()\n",
    "text = tokenizer.decode(generated_sequence, clean_up_tokenization_spaces=True)\n",
    "text"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f912ce0-08e5-44a8-8061-766c5bb2d70a",
   "metadata": {},
   "source": [
    "# Falcon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "04fb39a9-444a-466d-a08d-6d8f1262ec1a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please run\n",
      "\n",
      "python -m bitsandbytes\n",
      "\n",
      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "bin /data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32\n",
      "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.0\n",
      "CUDA SETUP: Detected CUDA version 117\n",
      "CUDA SETUP: Loading binary /data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable.\n",
      "  warn(\"The installed version of bitsandbytes was compiled without GPU support. \"\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/user/rajkumardheivanayahi/oauth_callback')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('servers!server=rajkumardheivanayahi/\", \"access'), PosixPath('[\"access'), PosixPath('servers!user=rajkumardheivanayahi\"]')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//127.0.0.1'), PosixPath('8081/hub/api'), PosixPath('http')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('8081/hub/api/users/rajkumardheivanayahi/activity'), PosixPath('//127.0.0.1'), PosixPath('http')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/user/rajkumardheivanayahi')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('49529/user/rajkumardheivanayahi'), PosixPath('//127.0.0.1'), PosixPath('http')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//matplotlib_inline.backend_inline')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/cuda/lib64')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: No libcudart.so found! Install CUDA or the cudatoolkit package (anaconda)!\n",
      "  warn(msg)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d51a2b3966c46a696b5da9b4f57b860",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "AttributeError",
     "evalue": "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cquantize_blockwise_fp16_nf4",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_1707867/632911219.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     10\u001b[0m )\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m model = AutoModelForCausalLM.from_pretrained(\n\u001b[0m\u001b[1;32m     13\u001b[0m     \u001b[0mmodel_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0mquantization_config\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbnb_config\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m    477\u001b[0m             )\n\u001b[1;32m    478\u001b[0m             \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhub_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"code_revision\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 479\u001b[0;31m             return model_class.from_pretrained(\n\u001b[0m\u001b[1;32m    480\u001b[0m                 \u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmodel_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mhub_kwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    481\u001b[0m             )\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m   2879\u001b[0m                 \u001b[0moffload_index\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2880\u001b[0m                 \u001b[0merror_msgs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2881\u001b[0;31m             \u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_load_pretrained_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2882\u001b[0m                 \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2883\u001b[0m                 \u001b[0mstate_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36m_load_pretrained_model\u001b[0;34m(cls, model, state_dict, loaded_keys, resolved_archive_file, pretrained_model_name_or_path, ignore_mismatched_sizes, sharded_metadata, _fast_init, low_cpu_mem_usage, device_map, offload_folder, offload_state_dict, dtype, is_quantized, keep_in_fp32_modules)\u001b[0m\n\u001b[1;32m   3226\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3227\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mlow_cpu_mem_usage\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3228\u001b[0;31m                     new_error_msgs, offload_index, state_dict_index = _load_state_dict_into_meta_model(\n\u001b[0m\u001b[1;32m   3229\u001b[0m                         \u001b[0mmodel_to_load\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3230\u001b[0m                         \u001b[0mstate_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36m_load_state_dict_into_meta_model\u001b[0;34m(model, state_dict, loaded_state_dict_keys, start_prefix, expected_keys, device_map, offload_folder, offload_index, state_dict_folder, state_dict_index, dtype, is_quantized, is_safetensors, keep_in_fp32_modules)\u001b[0m\n\u001b[1;32m    726\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    727\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;34m\"SCB\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mparam_name\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 728\u001b[0;31m                 set_module_quantized_tensor_to_device(\n\u001b[0m\u001b[1;32m    729\u001b[0m                     \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfp16_statistics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfp16_statistics\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    730\u001b[0m                 )\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/transformers/utils/bitsandbytes.py\u001b[0m in \u001b[0;36mset_module_quantized_tensor_to_device\u001b[0;34m(module, tensor_name, device, value, fp16_statistics)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 \u001b[0mnew_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbnb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mInt8Params\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequires_grad\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mis_4bit\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m                 \u001b[0mnew_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbnb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mParams4bit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequires_grad\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m             \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parameters\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtensor_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/bitsandbytes/nn/modules.py\u001b[0m in \u001b[0;36mto\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"cuda\"\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"cpu\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 178\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    179\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    180\u001b[0m             \u001b[0ms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquant_state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/bitsandbytes/nn/modules.py\u001b[0m in \u001b[0;36mcuda\u001b[0;34m(self, device)\u001b[0m\n\u001b[1;32m    154\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    155\u001b[0m         \u001b[0mw\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontiguous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhalf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 156\u001b[0;31m         \u001b[0mw_4bit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquant_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbnb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunctional\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquantize_4bit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mw\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblocksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mblocksize\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompress_statistics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompress_statistics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mquant_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquant_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    157\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mw_4bit\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    158\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquant_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mquant_state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/bitsandbytes/functional.py\u001b[0m in \u001b[0;36mquantize_4bit\u001b[0;34m(A, absmax, out, blocksize, compress_statistics, quant_type)\u001b[0m\n\u001b[1;32m    817\u001b[0m             \u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcquantize_blockwise_fp16_fp4\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mget_ptr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mget_ptr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mget_ptr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mabsmax\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mget_ptr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_int32\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblocksize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_int\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    818\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 819\u001b[0;31m             \u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcquantize_blockwise_fp16_nf4\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mget_ptr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mget_ptr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mget_ptr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mabsmax\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mget_ptr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_int32\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblocksize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_int\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    820\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbfloat16\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    821\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mquant_type\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'fp4'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/jupyter-hub/lib/python3.9/ctypes/__init__.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m    393\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'__'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mendswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'__'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    394\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 395\u001b[0;31m         \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    396\u001b[0m         \u001b[0msetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    397\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/jupyter-hub/lib/python3.9/ctypes/__init__.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, name_or_ordinal)\u001b[0m\n\u001b[1;32m    398\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    399\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname_or_ordinal\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 400\u001b[0;31m         \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_FuncPtr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_ordinal\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    401\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_ordinal\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    402\u001b[0m             \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname_or_ordinal\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: /data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cquantize_blockwise_fp16_nf4"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, AutoTokenizer\n",
    "\n",
    "model_name = \"tiiuae/falcon-7b\"\n",
    "\n",
    "bnb_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    bnb_4bit_quant_type=\"nf4\",\n",
    "    bnb_4bit_compute_dtype=torch.float16,\n",
    ")\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_name,\n",
    "    quantization_config=bnb_config,\n",
    "    trust_remote_code=True\n",
    ")\n",
    "model.config.use_cache = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5c047385-7150-425f-99da-afd74891a7a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/opt/jupyter-hub/bin/python'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.executable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e47dca3e-41d8-49b5-87fc-30f955a648be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'IP version: 4.0. IP ihl: 5.0. IP tos: 0x0. IP len: 219.0. IP frag: 0.0. IP ttl: 62.0. IP proto: tcp. IP src: 175.45.176.2. IP dst: 149.171.126.19. TCP sport: 19661. TCP dport: http. TCP seq: 1739521222.0. TCP ack: 578738309.0. TCP dataofs: 5.0. TCP reserved: 0.0. TCP flags: PA. TCP window: 16383.0. TCP chksum: 0x6ede. TCP urgptr: 0.0. TCP options: []. The payload contains these words: GET  GcI.asp HTTP Host  OpLqwnfYOUser Agent  Mozilla compatible  MSIE   Windows NT   Trident Accept Connection  keep aliveContent Length'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1['packet+payload'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d1688e9-4e51-4899-a127-9a8e628cbdc9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52481def-6bd8-44a5-a95d-c7673d66d713",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eddfcfc3-61ab-4d20-affe-1e7861cbd807",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9c59c2c8-10bb-43d2-bc44-632490718165",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a00850e853fb49f9b8c13a2e7c5bd661",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model_name = \"tiiuae/falcon-7b\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name, pad_token = tokenizer.eos_token)\n",
    "model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d6a431c8-4423-4aa6-92a1-ce148a47db29",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from datasets import Dataset, load_dataset\n",
    "df1 = pd.read_csv('./Text Generation/QUESTION.csv')\n",
    "df2 = pd.read_csv('./Text Generation/ANSWER.csv')\n",
    "df = pd.DataFrame({'Text': df1['packet+payload'] + \" ### \" + df2['explanation']})\n",
    "dataset = Dataset.from_pandas(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8f50fb4b-1f39-4d08-b064-3396657ec16b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/53611 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "trainer = SFTTrainer(\n",
    "    model,\n",
    "    tokenizer=tokenizer,\n",
    "    train_dataset=dataset,\n",
    "    dataset_text_field=\"Text\",\n",
    "    max_seq_length=512\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ebec574-6927-495b-a3ea-0d94faecc6ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "25e3e6cf-8a1c-4c6b-9017-e833c01a4b55",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please run\n",
      "\n",
      "python -m bitsandbytes\n",
      "\n",
      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "bin /data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32\n",
      "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.0\n",
      "CUDA SETUP: Detected CUDA version 117\n",
      "CUDA SETUP: Loading binary /data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable.\n",
      "  warn(\"The installed version of bitsandbytes was compiled without GPU support. \"\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/user/rajkumardheivanayahi/oauth_callback')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('servers!server=rajkumardheivanayahi/\", \"access'), PosixPath('servers!user=rajkumardheivanayahi\"]'), PosixPath('[\"access')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('http'), PosixPath('8081/hub/api'), PosixPath('//127.0.0.1')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('http'), PosixPath('8081/hub/api/users/rajkumardheivanayahi/activity'), PosixPath('//127.0.0.1')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/user/rajkumardheivanayahi')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('http'), PosixPath('49529/user/rajkumardheivanayahi'), PosixPath('//127.0.0.1')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//matplotlib_inline.backend_inline'), PosixPath('module')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/cuda/lib64')}\n",
      "  warn(msg)\n",
      "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: No libcudart.so found! Install CUDA or the cudatoolkit package (anaconda)!\n",
      "  warn(msg)\n"
     ]
    }
   ],
   "source": [
    "import torch, einops\n",
    "from datasets import load_dataset\n",
    "from peft import LoraConfig\n",
    "from transformers import (\n",
    "    AutoModelForCausalLM,\n",
    "    AutoTokenizer,\n",
    "    BitsAndBytesConfig,\n",
    "    AutoTokenizer,\n",
    "    TrainingArguments\n",
    ")\n",
    "from peft.tuners.lora import LoraLayer\n",
    "\n",
    "from trl import SFTTrainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f389c2ac-3787-4d9f-9f14-ffb68f4c3e64",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_and_prepare_model():\n",
    "    compute_dtype = getattr(torch, \"float16\")\n",
    "\n",
    "    bnb_config = BitsAndBytesConfig(\n",
    "        load_in_4bit=True,\n",
    "        bnb_4bit_quant_type=\"nf4\",\n",
    "        bnb_4bit_compute_dtype=compute_dtype,\n",
    "        bnb_4bit_use_double_quant=True,\n",
    "    )\n",
    "\n",
    "    model = AutoModelForCausalLM.from_pretrained(\n",
    "        \"tiiuae/falcon-7b\", trust_remote_code=True\n",
    "    )\n",
    "\n",
    "    peft_config = LoraConfig(\n",
    "        lora_alpha=16,\n",
    "        lora_dropout=0.1,\n",
    "        r=64,\n",
    "        bias=\"none\",\n",
    "        task_type=\"CAUSAL_LM\",\n",
    "        target_modules=[\n",
    "            \"query_key_value\"\n",
    "        ],\n",
    "    )\n",
    "\n",
    "    tokenizer = AutoTokenizer.from_pretrained(\"tiiuae/falcon-7b\", trust_remote_code=True)\n",
    "    tokenizer.pad_token = tokenizer.eos_token\n",
    "\n",
    "    return model, peft_config, tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c026518e-908b-403e-8235-a1a3459f8009",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_arguments = TrainingArguments(\n",
    "    output_dir=\"./results\",\n",
    "    per_device_train_batch_size=1,\n",
    "    gradient_accumulation_steps=4,\n",
    "    optim=\"paged_adamw_32bit\",\n",
    "    save_steps=10,\n",
    "    logging_steps=10,\n",
    "    learning_rate=2e-4,\n",
    "    fp16=True,\n",
    "    max_grad_norm=0.3,\n",
    "    max_steps=10000,\n",
    "    warmup_ratio=0.03,\n",
    "    group_by_length=True,\n",
    "    lr_scheduler_type=\"constant\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3f57bfda-c495-4a2f-9d7f-2a242be629e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd12a8e497974c0c9fc78ed5c2664c78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "ValueError",
     "evalue": "the `--group_by_length` option is only available for `Dataset`, not `IterableDataset",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_4092882/3012649071.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_cache\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m trainer = SFTTrainer(\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mtrain_dataset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/trl/trainer/sft_trainer.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, model, args, data_collator, train_dataset, eval_dataset, tokenizer, model_init, compute_metrics, callbacks, optimizers, preprocess_logits_for_metrics, peft_config, dataset_text_field, packing, formatting_func, max_seq_length, infinite, num_of_sequences, chars_per_token)\u001b[0m\n\u001b[1;32m    195\u001b[0m             )\n\u001b[1;32m    196\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 197\u001b[0;31m         super().__init__(\n\u001b[0m\u001b[1;32m    198\u001b[0m             \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m             \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, model, args, data_collator, train_dataset, eval_dataset, tokenizer, model_init, compute_metrics, callbacks, optimizers, preprocess_logits_for_metrics)\u001b[0m\n\u001b[1;32m    588\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroup_by_length\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    589\u001b[0m         ):\n\u001b[0;32m--> 590\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"the `--group_by_length` option is only available for `Dataset`, not `IterableDataset\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    591\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    592\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_signature_columns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: the `--group_by_length` option is only available for `Dataset`, not `IterableDataset"
     ]
    }
   ],
   "source": [
    "model, peft_config, tokenizer = create_and_prepare_model()\n",
    "model.config.use_cache = False\n",
    "\n",
    "trainer = SFTTrainer(\n",
    "    model=model,\n",
    "    train_dataset=dataset,\n",
    "    peft_config=peft_config,\n",
    "    dataset_text_field=\"Text\",\n",
    "    max_seq_length=512,\n",
    "    tokenizer=tokenizer,\n",
    "    args=training_arguments,\n",
    "    packing=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9715b095-0a50-4c1d-a74a-45f792b41650",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['Text'],\n",
       "    num_rows: 53611\n",
       "})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5fdae2d0-b956-4629-bb1b-725c2891c0cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset json (/data/rajkumardheivanayahi/.cache/huggingface/datasets/timdettmers___json/timdettmers--openassistant-guanaco-6126c710748182cf/0.0.0/e347ab1c932092252e717ff3f949105a4dd28b27e842dd53157d2f72e276c2e4)\n"
     ]
    }
   ],
   "source": [
    "dataset = load_dataset('timdettmers/openassistant-guanaco', split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ee7c968c-a781-454e-9cd1-43457d5ecffd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text'],\n",
       "    num_rows: 9846\n",
       "})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bf5573ec-2ab7-40d2-9ba2-2a65d36fe107",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import pipeline, AutoModelForCausalLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "26d88f3f-904b-4131-82a4-9d0414246704",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Loading tiiuae/falcon-7b requires to execute some code in that repo, you can inspect the content of the repository at https://hf.co/tiiuae/falcon-7b. You can dismiss this prompt by passing `trust_remote_code=True`.\n",
      "Do you accept? [y/N]  y\n",
      "Loading tiiuae/falcon-7b requires to execute some code in that repo, you can inspect the content of the repository at https://hf.co/tiiuae/falcon-7b. You can dismiss this prompt by passing `trust_remote_code=True`.\n",
      "Do you accept? [y/N]  y\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a2bb88878364e9da36b4016caeac8c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "AttributeError",
     "evalue": "/data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cget_col_row_stats",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_796311/319000902.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mpeft_model_id\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"./content/results/checkpoint-480\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mconfig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPeftConfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpeft_model_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAutoModelForCausalLM\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbase_model_name_or_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mload_in_8bit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice_map\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'auto'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0mtokenizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAutoTokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbase_model_name_or_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/transformers/models/auto/auto_factory.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m    477\u001b[0m             )\n\u001b[1;32m    478\u001b[0m             \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhub_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"code_revision\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 479\u001b[0;31m             return model_class.from_pretrained(\n\u001b[0m\u001b[1;32m    480\u001b[0m                 \u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmodel_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mhub_kwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    481\u001b[0m             )\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m   2879\u001b[0m                 \u001b[0moffload_index\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2880\u001b[0m                 \u001b[0merror_msgs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2881\u001b[0;31m             \u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_load_pretrained_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2882\u001b[0m                 \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2883\u001b[0m                 \u001b[0mstate_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36m_load_pretrained_model\u001b[0;34m(cls, model, state_dict, loaded_keys, resolved_archive_file, pretrained_model_name_or_path, ignore_mismatched_sizes, sharded_metadata, _fast_init, low_cpu_mem_usage, device_map, offload_folder, offload_state_dict, dtype, is_quantized, keep_in_fp32_modules)\u001b[0m\n\u001b[1;32m   3226\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3227\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mlow_cpu_mem_usage\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3228\u001b[0;31m                     new_error_msgs, offload_index, state_dict_index = _load_state_dict_into_meta_model(\n\u001b[0m\u001b[1;32m   3229\u001b[0m                         \u001b[0mmodel_to_load\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3230\u001b[0m                         \u001b[0mstate_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36m_load_state_dict_into_meta_model\u001b[0;34m(model, state_dict, loaded_state_dict_keys, start_prefix, expected_keys, device_map, offload_folder, offload_index, state_dict_folder, state_dict_index, dtype, is_quantized, is_safetensors, keep_in_fp32_modules)\u001b[0m\n\u001b[1;32m    726\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    727\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;34m\"SCB\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mparam_name\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 728\u001b[0;31m                 set_module_quantized_tensor_to_device(\n\u001b[0m\u001b[1;32m    729\u001b[0m                     \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfp16_statistics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfp16_statistics\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    730\u001b[0m                 )\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/transformers/utils/bitsandbytes.py\u001b[0m in \u001b[0;36mset_module_quantized_tensor_to_device\u001b[0;34m(module, tensor_name, device, value, fp16_statistics)\u001b[0m\n\u001b[1;32m     87\u001b[0m             \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mold_value\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__dict__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_8bit\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m                 \u001b[0mnew_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbnb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mInt8Params\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequires_grad\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     90\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mis_4bit\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m                 \u001b[0mnew_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbnb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mParams4bit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequires_grad\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/bitsandbytes/nn/modules.py\u001b[0m in \u001b[0;36mto\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    292\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"cpu\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    293\u001b[0m         ):\n\u001b[0;32m--> 294\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    295\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    296\u001b[0m             new_param = Int8Params(\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/bitsandbytes/nn/modules.py\u001b[0m in \u001b[0;36mcuda\u001b[0;34m(self, device)\u001b[0m\n\u001b[1;32m    256\u001b[0m             \u001b[0;31m# we convert this weight to the turning/ampere weight during the first inference pass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    257\u001b[0m             \u001b[0mB\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontiguous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhalf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 258\u001b[0;31m             \u001b[0mCB\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mCBt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSCB\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSCBt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcoo_tensorB\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbnb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunctional\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdouble_quant\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mB\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    259\u001b[0m             \u001b[0;32mdel\u001b[0m \u001b[0mCBt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m             \u001b[0;32mdel\u001b[0m \u001b[0mSCBt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/bitsandbytes/functional.py\u001b[0m in \u001b[0;36mdouble_quant\u001b[0;34m(A, col_stats, row_stats, out_col, out_row, threshold)\u001b[0m\n\u001b[1;32m   1985\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1986\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mrow_stats\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mcol_stats\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1987\u001b[0;31m         row_stats, col_stats, nnz_row_ptr = get_colrow_absmax(\n\u001b[0m\u001b[1;32m   1988\u001b[0m             \u001b[0mA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthreshold\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mthreshold\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1989\u001b[0m         )\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/bitsandbytes/functional.py\u001b[0m in \u001b[0;36mget_colrow_absmax\u001b[0;34m(A, row_stats, col_stats, nnz_block_ptr, threshold)\u001b[0m\n\u001b[1;32m   1874\u001b[0m     \u001b[0mprev_device\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpre_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1875\u001b[0m     \u001b[0mis_on_gpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrow_stats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcol_stats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnnz_block_ptr\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1876\u001b[0;31m     \u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcget_col_row_stats\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mptrA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mptrRowStats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mptrColStats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mptrNnzrows\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mct\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_float\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mthreshold\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrows\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcols\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1877\u001b[0m     \u001b[0mpost_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprev_device\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1878\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/jupyter-hub/lib/python3.9/ctypes/__init__.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m    393\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'__'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mendswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'__'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    394\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 395\u001b[0;31m         \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    396\u001b[0m         \u001b[0msetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    397\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/jupyter-hub/lib/python3.9/ctypes/__init__.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, name_or_ordinal)\u001b[0m\n\u001b[1;32m    398\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    399\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname_or_ordinal\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 400\u001b[0;31m         \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_FuncPtr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_ordinal\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    401\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_ordinal\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    402\u001b[0m             \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname_or_ordinal\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: /data/rajkumardheivanayahi/.local/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cget_col_row_stats"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from peft import PeftModel, PeftConfig\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "\n",
    "peft_model_id = \"./content/results/checkpoint-480\"\n",
    "config = PeftConfig.from_pretrained(peft_model_id)\n",
    "model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, return_dict=True, load_in_8bit=True, device_map='auto')\n",
    "tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n",
    "\n",
    "# Load the Lora model\n",
    "model = PeftModel.from_pretrained(model, peft_model_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d2693a8a-4a11-4c9a-8d65-ccaca01e47a2",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Unrecognized model in ./content/results/checkpoint-480. Should have a `model_type` key in its config.json, or contain one of the following strings in its name: albert, align, altclip, audio-spectrogram-transformer, autoformer, bart, beit, bert, bert-generation, big_bird, bigbird_pegasus, biogpt, bit, blenderbot, blenderbot-small, blip, blip-2, bloom, bridgetower, camembert, canine, chinese_clip, clap, clip, clipseg, codegen, conditional_detr, convbert, convnext, convnextv2, cpmant, ctrl, cvt, data2vec-audio, data2vec-text, data2vec-vision, deberta, deberta-v2, decision_transformer, deformable_detr, deit, deta, detr, dinat, distilbert, donut-swin, dpr, dpt, efficientformer, efficientnet, electra, encoder-decoder, ernie, ernie_m, esm, flaubert, flava, fnet, focalnet, fsmt, funnel, git, glpn, gpt-sw3, gpt2, gpt_bigcode, gpt_neo, gpt_neox, gpt_neox_japanese, gptj, gptsan-japanese, graphormer, groupvit, hubert, ibert, imagegpt, informer, jukebox, layoutlm, layoutlmv2, layoutlmv3, led, levit, lilt, llama, longformer, longt5, luke, lxmert, m2m_100, marian, markuplm, mask2former, maskformer, maskformer-swin, mbart, mctct, mega, megatron-bert, mgp-str, mobilebert, mobilenet_v1, mobilenet_v2, mobilevit, mobilevitv2, mpnet, mt5, mvp, nat, nezha, nllb-moe, nystromformer, oneformer, open-llama, openai-gpt, opt, owlvit, pegasus, pegasus_x, perceiver, pix2struct, plbart, poolformer, prophetnet, qdqbert, rag, realm, reformer, regnet, rembert, resnet, retribert, roberta, roberta-prelayernorm, roc_bert, roformer, rwkv, sam, segformer, sew, sew-d, speech-encoder-decoder, speech_to_text, speech_to_text_2, speecht5, splinter, squeezebert, swiftformer, swin, swin2sr, swinv2, switch_transformers, t5, table-transformer, tapas, time_series_transformer, timesformer, timm_backbone, trajectory_transformer, transfo-xl, trocr, tvlt, unispeech, unispeech-sat, upernet, van, videomae, vilt, vision-encoder-decoder, vision-text-dual-encoder, visual_bert, vit, vit_hybrid, vit_mae, vit_msn, wav2vec2, wav2vec2-conformer, wavlm, whisper, xclip, xglm, xlm, xlm-prophetnet, xlm-roberta, xlm-roberta-xl, xlnet, xmod, yolos, yoso",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_796311/2889580192.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpipe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpipeline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"text-generation\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"./content/results/checkpoint-480\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrust_remote_code\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/transformers/pipelines/__init__.py\u001b[0m in \u001b[0;36mpipeline\u001b[0;34m(task, model, config, tokenizer, feature_extractor, image_processor, framework, revision, use_fast, use_auth_token, device, device_map, torch_dtype, trust_remote_code, model_kwargs, pipeline_class, **kwargs)\u001b[0m\n\u001b[1;32m    703\u001b[0m         \u001b[0mhub_kwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"_commit_hash\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_commit_hash\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    704\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mconfig\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 705\u001b[0;31m         \u001b[0mconfig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAutoConfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_from_pipeline\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mhub_kwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mmodel_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    706\u001b[0m         \u001b[0mhub_kwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"_commit_hash\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_commit_hash\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    707\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.9/site-packages/transformers/models/auto/configuration_auto.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, **kwargs)\u001b[0m\n\u001b[1;32m    964\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mCONFIG_MAPPING\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpattern\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0munused_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    965\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 966\u001b[0;31m         raise ValueError(\n\u001b[0m\u001b[1;32m    967\u001b[0m             \u001b[0;34mf\"Unrecognized model in {pretrained_model_name_or_path}. \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    968\u001b[0m             \u001b[0;34mf\"Should have a `model_type` key in its {CONFIG_NAME}, or contain one of the following strings \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Unrecognized model in ./content/results/checkpoint-480. Should have a `model_type` key in its config.json, or contain one of the following strings in its name: albert, align, altclip, audio-spectrogram-transformer, autoformer, bart, beit, bert, bert-generation, big_bird, bigbird_pegasus, biogpt, bit, blenderbot, blenderbot-small, blip, blip-2, bloom, bridgetower, camembert, canine, chinese_clip, clap, clip, clipseg, codegen, conditional_detr, convbert, convnext, convnextv2, cpmant, ctrl, cvt, data2vec-audio, data2vec-text, data2vec-vision, deberta, deberta-v2, decision_transformer, deformable_detr, deit, deta, detr, dinat, distilbert, donut-swin, dpr, dpt, efficientformer, efficientnet, electra, encoder-decoder, ernie, ernie_m, esm, flaubert, flava, fnet, focalnet, fsmt, funnel, git, glpn, gpt-sw3, gpt2, gpt_bigcode, gpt_neo, gpt_neox, gpt_neox_japanese, gptj, gptsan-japanese, graphormer, groupvit, hubert, ibert, imagegpt, informer, jukebox, layoutlm, layoutlmv2, layoutlmv3, led, levit, lilt, llama, longformer, longt5, luke, lxmert, m2m_100, marian, markuplm, mask2former, maskformer, maskformer-swin, mbart, mctct, mega, megatron-bert, mgp-str, mobilebert, mobilenet_v1, mobilenet_v2, mobilevit, mobilevitv2, mpnet, mt5, mvp, nat, nezha, nllb-moe, nystromformer, oneformer, open-llama, openai-gpt, opt, owlvit, pegasus, pegasus_x, perceiver, pix2struct, plbart, poolformer, prophetnet, qdqbert, rag, realm, reformer, regnet, rembert, resnet, retribert, roberta, roberta-prelayernorm, roc_bert, roformer, rwkv, sam, segformer, sew, sew-d, speech-encoder-decoder, speech_to_text, speech_to_text_2, speecht5, splinter, squeezebert, swiftformer, swin, swin2sr, swinv2, switch_transformers, t5, table-transformer, tapas, time_series_transformer, timesformer, timm_backbone, trajectory_transformer, transfo-xl, trocr, tvlt, unispeech, unispeech-sat, upernet, van, videomae, vilt, vision-encoder-decoder, vision-text-dual-encoder, visual_bert, vit, vit_hybrid, vit_mae, vit_msn, wav2vec2, wav2vec2-conformer, wavlm, whisper, xclip, xglm, xlm, xlm-prophetnet, xlm-roberta, xlm-roberta-xl, xlnet, xmod, yolos, yoso"
     ]
    }
   ],
   "source": [
    "pipe = pipeline(\"text-generation\", model=\"./content/results/checkpoint-480\", trust_remote_code=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
